Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

feat(pipelines): Add UI selector for choosing which tables to skip copy of (#47808)

Riccardo Busettib2b150fa3cfa
diff --git a/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.test.tsx b/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.test.tsx
new file mode 100644
index 00000000000..cae640ff9a7
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.test.tsx
@@ -0,0 +1,81 @@
+import { act, fireEvent, render, screen } from '@testing-library/react'
+import { describe, expect, it, vi } from 'vitest'
+
+import { BatchRestartDialog } from './BatchRestartDialog'
+import type { ReplicationPipelineTableStatus } from '@/data/replication/pipeline-replication-status-query'
+
+const mocks = vi.hoisted(() => ({
+ rollbackTables: vi.fn().mockResolvedValue({ pipeline_id: 9, tables: [] }),
+}))
+
+vi.mock('common', () => ({
+ useParams: () => ({ ref: 'project-ref', pipelineId: '9' }),
+}))
+vi.mock('@/data/replication/rollback-tables-mutation', () => ({
+ useRollbackTablesMutation: () => ({
+ mutateAsync: mocks.rollbackTables,
+ isPending: false,
+ }),
+}))
+vi.mock('./RestartCostEstimate', () => ({
+ RestartCostEstimate: ({ tables }: { tables: { schema: string; name: string }[] }) => (
+ <div data-testid="copy-targets">
+ {tables.map(({ schema, name }) => `${schema}.${name}`).join(',')}
+ </div>
+ ),
+}))
+
+const table = (
+ id: number,
+ state: ReplicationPipelineTableStatus['state']
+): ReplicationPipelineTableStatus => ({
+ id,
+ schema: 'public',
+ name: `table_${id}`,
+ table_id: id,
+ table_name: `public.table_${id}`,
+ state,
+})
+
+describe('BatchRestartDialog', () => {
+ it('describes every table reset by the all-errored backend target', async () => {
+ const onRestartStart = vi.fn()
+ const tables = [
+ table(1, { name: 'error', reason: 'manual', retry_policy: { policy: 'manual_retry' } }),
+ table(2, { name: 'error', reason: 'terminal', retry_policy: { policy: 'no_retry' } }),
+ table(3, {
+ name: 'error',
+ reason: 'timed',
+ retry_policy: { policy: 'timed_retry', next_retry: '2026-07-21T12:00:00Z' },
+ }),
+ table(4, { name: 'following_wal' }),
+ ]
+
+ render(
+ <BatchRestartDialog
+ open
+ onOpenChange={vi.fn()}
+ mode="errored"
+ tables={tables}
+ tableSyncCopy={{ type: 'include_tables', table_ids: [1, 2] }}
+ onRestartStart={onRestartStart}
+ />
+ )
+
+ expect(screen.getByText(/3 currently failed tables/)).toBeInTheDocument()
+ expect(screen.getByTestId('copy-targets')).toHaveTextContent('public.table_1,public.table_2')
+
+ await act(async () => {
+ fireEvent.click(screen.getByRole('button', { name: 'Restart failed tables' }))
+ })
+
+ expect(onRestartStart).toHaveBeenCalledWith([1, 2, 3])
+ expect(mocks.rollbackTables).toHaveBeenCalledWith(
+ expect.objectContaining({
+ pipelineId: 9,
+ target: { type: 'all_errored_tables' },
+ rollbackType: 'full',
+ })
+ )
+ })
+})
diff --git a/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsx b/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsx
index 5ee178156e6..134849a32f0 100644
--- a/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsx
@@ -14,6 +14,7 @@ import {
import { PipelineStatusName } from './Replication.constants'
import { RestartCostEstimate } from './RestartCostEstimate'
+import { getTableCopyTargets, type TableSyncCopyConfig } from './TableSyncCopy.utils'
import { ReplicationPipelineTableStatus } from '@/data/replication/pipeline-replication-status-query'
import { useRollbackTablesMutation } from '@/data/replication/rollback-tables-mutation'
@@ -24,6 +25,7 @@ interface BatchRestartDialogProps {
tables: ReplicationPipelineTableStatus[]
sourceId?: number
publicationName?: string
+ tableSyncCopy?: TableSyncCopyConfig
pipelineStatusName?: PipelineStatusName
onRestartStart?: (tableIds: number[]) => void
onRestartComplete?: (tableIds: number[]) => void
@@ -36,6 +38,7 @@ export const BatchRestartDialog = ({
tables,
sourceId,
publicationName,
+ tableSyncCopy,
pipelineStatusName,
onRestartStart,
onRestartComplete,
@@ -46,24 +49,41 @@ export const BatchRestartDialog = ({
if (mode === 'all') {
return tables
} else {
- return tables.filter(
- (t) =>
- t.state.name === 'error' &&
- 'retry_policy' in t.state &&
- t.state.retry_policy?.policy === 'manual_retry'
- )
+ return tables.filter((table) => table.state.name === 'error')
}
}, [mode, tables])
- const affectedTableIds = useMemo(
- () => affectedTables.map((table) => table.table_id),
- [affectedTables]
- )
+ const affectedTableIds = useMemo(() => affectedTables.map((table) => table.id), [affectedTables])
- const affectedTableNames = useMemo(
- () => affectedTables.map((table) => table.table_name),
- [affectedTables]
+ const copiedTables = useMemo(
+ () => getTableCopyTargets(affectedTables, tableSyncCopy),
+ [affectedTables, tableSyncCopy]
)
+ const initialCopyDescription =
+ copiedTables.length === 0 ? (
+ <li>
+ <strong>No table will run an initial copy.</strong> New changes will resume streaming
+ without backfilling existing source rows. There is no additional initial-copy charge.
+ </li>
+ ) : copiedTables.length === affectedTables.length ? (
+ <li>
+ <strong>
+ {copiedTables.length === 1 ? 'The table' : `All ${copiedTables.length} tables`} will run
+ an initial copy.
+ </strong>{' '}
+ Existing rows will be copied again from the source and billed in addition to previous
+ initial copies.
+ </li>
+ ) : (
+ <li>
+ <strong>
+ {copiedTables.length} of {affectedTables.length} tables will run an initial copy.
+ </strong>{' '}
+ The remaining tables will resume streaming without a backfill. Copied rows are billed in
+ addition to previous initial copies.
+ </li>
+ )
+
const { mutateAsync: rollbackTables, isPending: isResetting } = useRollbackTablesMutation({
onSuccess: (data) => {
const count = data.tables.length
@@ -107,11 +127,7 @@ export const BatchRestartDialog = ({
{affectedTables.length === 1 ? '' : 's'} in this pipeline from scratch:
</p>
<ul className="list-disc list-inside space-y-1.5 pl-2">
- <li>
- <strong>Every table's initial sync will restart.</strong> All table data will be
- copied again from the source. This initial sync is billed in addition to previous
- initial syncs.
- </li>
+ {initialCopyDescription}
<li>
<strong>All downstream data will be deleted.</strong> All replicated data will be
removed.
@@ -130,22 +146,18 @@ export const BatchRestartDialog = ({
description: (
<div className="space-y-3 text-sm">
<p>
- This will restart replication for{' '}
- <strong>all {affectedTables.length} failed tables</strong> from scratch:
+ This will restart replication for all{' '}
+ <strong>{affectedTables.length} currently failed tables</strong> from scratch:
</p>
<ul className="list-disc list-inside space-y-1.5 pl-2">
- <li>
- <strong>Each failed table's initial sync will restart.</strong> These tables will
- be copied again from the source. This initial sync is billed in addition to
- previous initial syncs.
- </li>
+ {initialCopyDescription}
<li>
<strong>Existing downstream data will be deleted.</strong> Replicated data for
these tables will be removed.
</li>
<li>
- <strong>All other tables remain untouched.</strong> Only failed tables are
- affected.
+ <strong>Tables that are not failed remain untouched.</strong> The request resets
+ every table that is failed when it runs.
</li>
<li>
<strong>The pipeline will restart automatically.</strong> This is required to
@@ -169,7 +181,7 @@ export const BatchRestartDialog = ({
projectRef={projectRef}
sourceId={sourceId}
publicationName={publicationName}
- tableNames={affectedTableNames}
+ tables={copiedTables}
/>
<AlertDialogFooter>
<AlertDialogCancel disabled={isResetting}>Cancel</AlertDialogCancel>
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/Fields.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/Fields.tsx
index eb674d27ea5..51c0746c7ac 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/Fields.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AnalyticsBucket/Fields.tsx
@@ -1,4 +1,3 @@
-import { PermissionAction } from '@supabase/shared-types/out/constants'
import { useParams } from 'common'
import { Eye, EyeOff, Loader2 } from 'lucide-react'
import { useState } from 'react'
@@ -27,11 +26,9 @@ import {
} from '../DestinationForm.constants'
import type { DestinationPanelSchemaType } from '../DestinationForm.schema'
import { InlineLink } from '@/components/ui/InlineLink'
-import { useAPIKeys } from '@/data/api-keys/api-keys-query'
import { useAnalyticsBucketsQuery } from '@/data/storage/analytics-buckets-query'
import { useIcebergNamespacesQuery } from '@/data/storage/iceberg-namespaces-query'
import { useStorageCredentialsQuery } from '@/data/storage/s3-access-key-query'
-import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions'
/**
* [Joshen] JFYI I'd foresee a possible UX friction point here regarding S3 access key IDs and secret access keys
@@ -70,14 +67,6 @@ export const AnalyticsBucketFields = ({
const { ref: projectRef } = useParams()
- const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')
- const { data: apiKeysData } = useAPIKeys(
- { projectRef, reveal: true },
- { enabled: canReadAPIKeys }
- )
- const { serviceKey } = apiKeysData ?? {}
- const serviceApiKey = serviceKey?.api_key ?? ''
-
const {
data: keysData,
isSuccess: isSuccessKeys,
@@ -96,7 +85,7 @@ export const AnalyticsBucketFields = ({
isError: isErrorBuckets,
} = useAnalyticsBucketsQuery({ projectRef })
- const canSelectNamespace = !!warehouseName && !!serviceApiKey
+ const canSelectNamespace = !!warehouseName
const {
data: namespaces = [],
@@ -104,7 +93,7 @@ export const AnalyticsBucketFields = ({
isError: isErrorNamespaces,
} = useIcebergNamespacesQuery(
{ projectRef, warehouse: warehouseName },
- { enabled: !!serviceApiKey }
+ { enabled: !!warehouseName }
)
return (
@@ -293,7 +282,7 @@ export const AnalyticsBucketFields = ({
type={showCatalogToken ? 'text' : 'password'}
placeholder={editMode ? STORED_SECRET_PLACEHOLDER : 'Auto-populated'}
actions={
- serviceApiKey ? (
+ field.value ? (
<div className="flex items-center justify-center">
<Button
variant="default"
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.test.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.test.ts
index d9418bc5723..374f82c22c2 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.test.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.test.ts
@@ -5,6 +5,8 @@ import { DestinationPanelFormSchema } from './DestinationForm.schema'
const requiredFields = {
name: 'BigQuery warehouse',
publicationName: 'analytics_publication',
+ tableSyncCopyMode: 'include_all_tables' as const,
+ tableSyncCopyTableIds: [],
}
describe('DestinationPanelFormSchema', () => {
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts
index 6832876589e..9bfdd084c6b 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts
@@ -9,6 +9,13 @@ import { SnowflakeFormSchema } from './Snowflake/Snowflake.schema'
const CommonFormSchema = z.object({
name: z.string().min(1, 'Name is required'),
publicationName: z.string().min(1, 'Publication is required'),
+ tableSyncCopyMode: z.enum([
+ 'include_all_tables',
+ 'skip_all_tables',
+ 'include_tables',
+ 'skip_tables',
+ ]),
+ tableSyncCopyTableIds: z.array(z.string()),
maxFillMs: z
.number()
.int('Batch wait time must be a whole number of milliseconds')
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts
index a23088a14a3..3c55a6a9de0 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts
@@ -5,15 +5,23 @@ import { getBigQueryValidationIssues } from './BigQuery/BigQuery.utils'
import { getClickHouseValidationIssues } from './ClickHouse/ClickHouse.utils'
import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
import {
+ buildBatchConfig,
buildDestinationConfig,
buildDestinationConfigForValidation,
+ buildTableSyncCopyConfig,
+ generateDefaultValues,
+ pruneStaleSelectedTableIds,
} from './DestinationForm.utils'
import { getDucklakeValidationIssues } from './DuckLake/DuckLake.utils'
import { getSnowflakeValidationIssues } from './Snowflake/Snowflake.utils'
+import type { ReplicationPipelineByIdData } from '@/data/replication/pipeline-by-id-query'
+import type { ReplicationPublication } from '@/data/replication/publications-query'
const baseDucklakeFormData = {
name: 'DuckLake Destination',
publicationName: 'pub',
+ tableSyncCopyMode: 'include_all_tables' as const,
+ tableSyncCopyTableIds: [],
maxFillMs: undefined,
maxTableSyncWorkers: undefined,
maxCopyConnectionsPerTable: undefined,
@@ -45,6 +53,8 @@ const baseDucklakeFormData = {
const baseSnowflakeFormData = {
name: 'Snowflake Destination',
publicationName: 'pub',
+ tableSyncCopyMode: 'include_all_tables' as const,
+ tableSyncCopyTableIds: [],
maxFillMs: undefined,
maxTableSyncWorkers: undefined,
maxCopyConnectionsPerTable: undefined,
@@ -80,9 +90,120 @@ const baseSnowflakeFormData = {
snowflakeRole: ' PIPELINES_ROLE ',
}
+describe('DestinationForm.utils table copy selection', () => {
+ it('preserves server-managed batch fields while changing the exposed fill interval', () => {
+ expect(
+ buildBatchConfig({
+ maxFillMs: 500,
+ existingBatch: {
+ max_fill_ms: 200,
+ max_bytes: 8_388_608,
+ memory_budget_ratio: 0.2,
+ },
+ })
+ ).toEqual({
+ maxFillMs: 500,
+ maxBytes: 8_388_608,
+ memoryBudgetRatio: 0.2,
+ })
+ })
+
+ it.each(['include_all_tables', 'skip_all_tables'] as const)(
+ 'builds the %s config without table ids',
+ (mode) => {
+ expect(
+ buildTableSyncCopyConfig({
+ mode,
+ selectedTableIds: [],
+ })
+ ).toEqual({ type: mode })
+ }
+ )
+
+ it.each(['include_tables', 'skip_tables'] as const)('uses selected table ids for %s', (mode) => {
+ expect(
+ buildTableSyncCopyConfig({
+ mode,
+ selectedTableIds: ['202', '101'],
+ })
+ ).toEqual({ type: mode, table_ids: [202, 101] })
+ })
+
+ it('rejects an invalid selected table id', () => {
+ expect(() =>
+ buildTableSyncCopyConfig({
+ mode: 'skip_tables',
+ selectedTableIds: ['not-an-id'],
+ })
+ ).toThrow('The selected table IDs are invalid')
+ })
+
+ it('hydrates configured pipeline table ids as selected table ids in edit mode', () => {
+ const pipelineData = {
+ config: {
+ publication_name: 'analytics',
+ table_sync_copy: {
+ type: 'include_tables',
+ table_ids: [202, 101],
+ },
+ },
+ } as unknown as ReplicationPipelineByIdData
+
+ const defaults = generateDefaultValues({
+ pipelineData,
+ catalogToken: '',
+ editMode: true,
+ })
+
+ expect(defaults.tableSyncCopyMode).toBe('include_tables')
+ expect(defaults.tableSyncCopyTableIds).toEqual(['202', '101'])
+ })
+
+ it('never hydrates an edit form with a newly revealed catalog token', () => {
+ const defaults = generateDefaultValues({
+ catalogToken: 'newly-revealed-token',
+ editMode: true,
+ })
+
+ expect(defaults.catalogToken).toBe('')
+ })
+
+ it('drops selected ids that are no longer in the publication', () => {
+ const publications = [
+ { name: 'analytics', tables: [{ id: 101, schema: 'public', name: 'orders' }] },
+ ] as ReplicationPublication[]
+
+ expect(
+ pruneStaleSelectedTableIds({
+ mode: 'include_tables',
+ selectedTableIds: ['101', '202'],
+ publications,
+ publicationName: 'analytics',
+ })
+ ).toEqual(['101'])
+ })
+
+ it('leaves selected ids untouched for non-selective modes', () => {
+ const publications = [
+ { name: 'analytics', tables: [{ id: 101, schema: 'public', name: 'orders' }] },
+ ] as ReplicationPublication[]
+
+ expect(
+ pruneStaleSelectedTableIds({
+ mode: 'include_all_tables',
+ selectedTableIds: ['202'],
+ publications,
+ publicationName: 'analytics',
+ })
+ ).toEqual(['202'])
+ })
+})
+
const baseClickHouseFormData = {
name: 'ClickHouse Destination',
publicationName: 'pub',
+ tableSyncCopyMode: 'include_all_tables' as const,
+ tableSyncCopyTableIds: [],
maxFillMs: undefined,
maxTableSyncWorkers: undefined,
maxCopyConnectionsPerTable: undefined,
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
index e62f6140718..a4b51e46829 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
@@ -25,6 +25,7 @@ import {
import { type DucklakeApiConfig } from './DuckLake/DuckLake.utils'
import { type SnowflakeApiConfig } from './Snowflake/Snowflake.utils'
import {
+ BatchConfig,
BigQueryDestinationConfig,
ClickHouseDestinationConfig,
DestinationConfig,
@@ -33,9 +34,11 @@ import {
DucklakeSupabaseDestinationConfig,
IcebergDestinationConfig,
SnowflakeDestinationConfig,
+ TableSyncCopyConfig,
} from '@/data/replication/create-destination-pipeline-mutation'
import { type ReplicationDestinationByIdData } from '@/data/replication/destination-by-id-query'
import { type ReplicationPipelineByIdData } from '@/data/replication/pipeline-by-id-query'
+import { type ReplicationPublication } from '@/data/replication/publications-query'
import { type ValidationFailure } from '@/data/replication/validate-destination-mutation'
import {
type CreateS3AccessKeyCredentialVariables,
@@ -72,37 +75,31 @@ export const generateDefaultValues = ({
editMode: boolean
}): DestinationPanelSchemaType => {
const config = destinationData?.config
- const isBigQueryConfig = config && 'big_query' in config
- const isIcebergConfig = config && 'iceberg' in config
- const ducklakeConfigValue =
- config && 'ducklake' in (config as Record<string, unknown>)
- ? (config as Record<string, unknown>).ducklake
+
+ const bigQueryConfig = config && 'big_query' in config ? config.big_query : undefined
+ const icebergConfig =
+ config && 'iceberg' in config && 'supabase' in config.iceberg
+ ? config.iceberg.supabase
: undefined
const ducklakeConfig =
- ducklakeConfigValue && typeof ducklakeConfigValue === 'object'
- ? (ducklakeConfigValue as DucklakeApiConfig)
- : undefined
- const snowflakeConfigValue =
- config && 'snowflake' in (config as Record<string, unknown>)
- ? (config as Record<string, unknown>).snowflake
- : undefined
+ config && 'ducklake' in config ? (config.ducklake as DucklakeApiConfig) : undefined
const snowflakeConfig =
- snowflakeConfigValue && typeof snowflakeConfigValue === 'object'
- ? (snowflakeConfigValue as SnowflakeApiConfig)
- : undefined
- const clickhouseConfigValue =
- config && 'clickhouse' in (config as Record<string, unknown>)
- ? (config as Record<string, unknown>).clickhouse
- : undefined
+ config && 'snowflake' in config ? (config.snowflake as SnowflakeApiConfig) : undefined
const clickhouseConfig =
- clickhouseConfigValue && typeof clickhouseConfigValue === 'object'
- ? (clickhouseConfigValue as ClickHouseApiConfig)
- : undefined
+ config && 'clickhouse' in config ? (config.clickhouse as ClickHouseApiConfig) : undefined
+
+ const tableSyncCopy = pipelineData?.config.table_sync_copy ?? {
+ type: 'include_all_tables' as const,
+ }
+ const tableSyncCopyTableIds =
+ 'table_ids' in tableSyncCopy ? tableSyncCopy.table_ids.map((id) => String(id)) : []
return {
// Common fields
name: destinationData?.name ?? '',
publicationName: pipelineData?.config.publication_name ?? '',
+ tableSyncCopyMode: tableSyncCopy.type,
+ tableSyncCopyTableIds,
maxFillMs: pipelineData?.config?.batch?.max_fill_ms ?? DEFAULT_MAX_FILL_MS,
maxTableSyncWorkers:
pipelineData?.config?.max_table_sync_workers ?? DEFAULT_MAX_TABLE_SYNC_WORKERS,
@@ -113,21 +110,23 @@ export const generateDefaultValues = ({
(pipelineData?.config as { invalidated_slot_behavior?: 'error' | 'recreate' } | undefined)
?.invalidated_slot_behavior ?? undefined,
// BigQuery fields
- projectId: isBigQueryConfig ? config.big_query.project_id : '',
- datasetId: isBigQueryConfig ? config.big_query.dataset_id : '',
- serviceAccountKey: isBigQueryConfig ? config.big_query.service_account_key : '',
+ projectId: bigQueryConfig?.project_id ?? '',
+ datasetId: bigQueryConfig?.dataset_id ?? '',
+ // Destination response DTOs intentionally omit stored secrets. Edit submissions
+ // leave blank secret fields unset so the existing values are preserved.
+ serviceAccountKey: '',
connectionPoolSize:
(config as { big_query?: { connection_pool_size?: number } } | undefined)?.big_query
?.connection_pool_size ?? DEFAULT_CONNECTION_POOL_SIZE,
- maxStalenessMins: isBigQueryConfig ? config.big_query.max_staleness_mins : undefined, // Default: null
+ maxStalenessMins: bigQueryConfig?.max_staleness_mins ?? undefined, // Default: null
// Analytics Bucket fields
- warehouseName: isIcebergConfig ? config.iceberg.supabase.warehouse_name : '',
- namespace: isIcebergConfig ? config.iceberg.supabase.namespace : '',
+ warehouseName: icebergConfig?.warehouse_name ?? '',
+ namespace: icebergConfig?.namespace ?? '',
newNamespaceName: '',
- catalogToken: isIcebergConfig ? config.iceberg.supabase.catalog_token : catalogToken,
- s3AccessKeyId: isIcebergConfig ? config.iceberg.supabase.s3_access_key_id : '',
- s3SecretAccessKey: isIcebergConfig ? config.iceberg.supabase.s3_secret_access_key : '',
- s3Region: region ?? (isIcebergConfig ? config.iceberg.supabase.s3_region : ''),
+ catalogToken: editMode || icebergConfig ? '' : catalogToken,
+ s3AccessKeyId: '',
+ s3SecretAccessKey: '',
+ s3Region: region ?? icebergConfig?.s3_region ?? '',
// DuckLake fields
// New destinations default to the managed "Use Supabase" mode with the current project
// pre-selected as both catalog and storage. Existing destinations always read back as the
@@ -163,6 +162,78 @@ export const generateDefaultValues = ({
}
}
+export const buildTableSyncCopyConfig = ({
+ mode,
+ selectedTableIds,
+}: {
+ mode: DestinationPanelSchemaType['tableSyncCopyMode']
+ selectedTableIds: string[]
+}): TableSyncCopyConfig => {
+ if (mode === 'include_all_tables' || mode === 'skip_all_tables') return { type: mode }
+
+ if (selectedTableIds.length === 0) {
+ throw new Error('Select at least one table for the initial copy')
+ }
+
+ const tableIds = selectedTableIds.map(Number)
+ if (tableIds.some((id) => !Number.isSafeInteger(id) || id <= 0)) {
+ throw new Error('The selected table IDs are invalid. Refresh and try again.')
+ }
+
+ return {
+ type: mode,
+ table_ids: tableIds,
+ }
+}
+
+export const buildBatchConfig = ({
+ maxFillMs,
+ existingBatch,
+}: {
+ maxFillMs?: number
+ existingBatch?: ReplicationPipelineByIdData['config']['batch']
+}): BatchConfig | undefined => {
+ if (maxFillMs === undefined && existingBatch === undefined) return undefined
+
+ return {
+ maxFillMs,
+ maxBytes: existingBatch?.max_bytes,
+ memoryBudgetRatio: existingBatch?.memory_budget_ratio,
+ }
+}
+
+// The set of table ids (as strings, matching form state) currently in the
+// selected publication. Selective table-copy only ever offers/keeps ids that
+// are in this set; ids selected previously that fall out of it are dropped at
+// submit time rather than resolved or displayed.
+export const getPublicationTableIds = (
+ publications: ReplicationPublication[],
+ publicationName: string
+): Set<string> => {
+ const publication = publications.find(({ name }) => name === publicationName)
+ return new Set((publication?.tables ?? []).map(({ id }) => String(id)))
+}
+
+// Drops selected table ids that are no longer in the current publication.
+// Create validation rejects ids that aren't published, while edits are checked
+// against the latest loaded publication before submitting.
+export const pruneStaleSelectedTableIds = ({
+ mode,
+ selectedTableIds,
+ publications,
+ publicationName,
+}: {
+ mode: DestinationPanelSchemaType['tableSyncCopyMode']
+ selectedTableIds: string[]
+ publications: ReplicationPublication[]
+ publicationName: string
+}): string[] => {
+ if (mode === 'include_all_tables' || mode === 'skip_all_tables') return selectedTableIds
+
+ const publicationTableIds = getPublicationTableIds(publications, publicationName)
+ return selectedTableIds.filter((id) => publicationTableIds.has(id))
+}
+
const buildBigQueryConfig = (
data: z.infer<typeof DestinationPanelFormSchema>
): BigQueryDestinationConfig => ({
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsx
index 8cb49e591e3..9e5ca2799dd 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsx
@@ -21,20 +21,21 @@ import { MultiSelector } from 'ui-patterns/multi-select'
import { z } from 'zod'
import { useCreatePublicationMutation } from '@/data/replication/publication-create-mutation'
+import { useReplicationSourceId } from '@/data/replication/sources-query'
import { useReplicationTablesQuery } from '@/data/replication/tables-query'
import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject'
interface NewPublicationPanelProps {
visible: boolean
- sourceId?: number
onClose: (newPublication?: string) => void
}
-export const NewPublicationPanel = ({ visible, sourceId, onClose }: NewPublicationPanelProps) => {
+export const NewPublicationPanel = ({ visible, onClose }: NewPublicationPanelProps) => {
const { ref: projectRef } = useParams()
const { data: project } = useSelectedProjectQuery()
+ const sourceId = useReplicationSourceId({ projectRef })
- const { data: tables } = useReplicationTablesQuery({ projectRef, sourceId })
+ const { data: tables } = useReplicationTablesQuery({ projectRef, sourceId }, { enabled: visible })
const { mutate: createPublication, isPending: creatingPublication } =
useCreatePublicationMutation({
@@ -81,82 +82,80 @@ export const NewPublicationPanel = ({ visible, sourceId, onClose }: NewPublicati
}
return (
- <>
- <Sheet open={visible} onOpenChange={() => onClose()}>
- <SheetContent size="default">
- <div className="flex flex-col h-full">
- <SheetHeader>
- <SheetTitle>Create a new publication</SheetTitle>
- <SheetDescription>Choose which tables to replicate to destinations.</SheetDescription>
- </SheetHeader>
- <SheetSection className="grow overflow-auto">
- <Form {...form}>
- <form
- id={formId}
- onSubmit={form.handleSubmit(onSubmit)}
- className="flex flex-col gap-y-4"
- >
- <FormField
- control={form.control}
- name="name"
- render={({ field }) => (
- <FormItemLayout label="Name" layout="vertical">
- <FormControl>
- <Input {...field} placeholder="Name" />
- </FormControl>
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="tables"
- render={({ field }) => (
- <FormItemLayout
- label="Tables"
- description="Select at least one table to include in the publication."
- >
- <FormControl>
- <MultiSelector
- values={field.value}
- onValuesChange={field.onChange}
- disabled={creatingPublication}
- >
- <MultiSelector.Trigger
- badgeLimit="wrap"
- label="Select tables..."
- mode="inline-combobox"
- />
- <MultiSelector.Content>
- <MultiSelector.List>
- {tables?.map((table) => (
- <MultiSelector.Item
- key={`${table.schema}.${table.name}`}
- value={`${table.schema}.${table.name}`}
- >
- {`${table.schema}.${table.name}`}
- </MultiSelector.Item>
- ))}
- </MultiSelector.List>
- </MultiSelector.Content>
- </MultiSelector>
- </FormControl>
- </FormItemLayout>
- )}
- />
- </form>
- </Form>
- </SheetSection>
- <SheetFooter>
- <Button variant="default" disabled={creatingPublication} onClick={() => onClose()}>
- Cancel
- </Button>
- <Button variant="primary" loading={creatingPublication} form={formId} type="submit">
- Create publication
- </Button>
- </SheetFooter>
- </div>
- </SheetContent>
- </Sheet>
- </>
+ <Sheet open={visible} onOpenChange={() => onClose()}>
+ <SheetContent size="default">
+ <div className="flex flex-col h-full">
+ <SheetHeader>
+ <SheetTitle>Create a new publication</SheetTitle>
+ <SheetDescription>Choose which tables to replicate to destinations.</SheetDescription>
+ </SheetHeader>
+ <SheetSection className="grow overflow-auto">
+ <Form {...form}>
+ <form
+ id={formId}
+ onSubmit={form.handleSubmit(onSubmit)}
+ className="flex flex-col gap-y-4"
+ >
+ <FormField
+ control={form.control}
+ name="name"
+ render={({ field }) => (
+ <FormItemLayout label="Name" layout="vertical">
+ <FormControl>
+ <Input {...field} placeholder="Name" />
+ </FormControl>
+ </FormItemLayout>
+ )}
+ />
+ <FormField
+ control={form.control}
+ name="tables"
+ render={({ field }) => (
+ <FormItemLayout
+ label="Tables"
+ description="Select at least one table to include in the publication."
+ >
+ <FormControl>
+ <MultiSelector
+ values={field.value}
+ onValuesChange={field.onChange}
+ disabled={creatingPublication}
+ >
+ <MultiSelector.Trigger
+ badgeLimit="wrap"
+ label="Select tables..."
+ mode="inline-combobox"
+ />
+ <MultiSelector.Content>
+ <MultiSelector.List>
+ {tables?.map((table) => (
+ <MultiSelector.Item
+ key={`${table.schema}.${table.name}`}
+ value={`${table.schema}.${table.name}`}
+ >
+ {`${table.schema}.${table.name}`}
+ </MultiSelector.Item>
+ ))}
+ </MultiSelector.List>
+ </MultiSelector.Content>
+ </MultiSelector>
+ </FormControl>
+ </FormItemLayout>
+ )}
+ />
+ </form>
+ </Form>
+ </SheetSection>
+ <SheetFooter>
+ <Button variant="default" disabled={creatingPublication} onClick={() => onClose()}>
+ Cancel
+ </Button>
+ <Button variant="primary" loading={creatingPublication} form={formId} type="submit">
+ Create publication
+ </Button>
+ </SheetFooter>
+ </div>
+ </SheetContent>
+ </Sheet>
)
}
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PipelineCostDialog.test.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PipelineCostDialog.test.tsx
new file mode 100644
index 00000000000..562b005dadb
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PipelineCostDialog.test.tsx
@@ -0,0 +1,121 @@
+import { screen } from '@testing-library/react'
+import type { components } from 'api-types'
+import { HttpResponse } from 'msw'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { PipelineCostDialog } from './PipelineCostDialog'
+import { customRender } from '@/tests/lib/custom-render'
+import { addAPIMock } from '@/tests/lib/msw'
+
+type CostEstimateResponse = components['schemas']['CostEstimateResponse']
+type ReplicationSourcesResponse = components['schemas']['ReplicationSourcesResponse']
+
+const tables = [
+ {
+ schema: 'public',
+ name: 'orders',
+ estimated_bytes: 600_000_000,
+ estimated_cost: 0.6,
+ is_row_filtered: false,
+ },
+ {
+ schema: 'billing',
+ name: 'invoices',
+ estimated_bytes: 9_400_000_000,
+ estimated_cost: 9.4,
+ is_row_filtered: false,
+ },
+]
+
+let costEstimateTables = tables
+
+const mockSources: ReplicationSourcesResponse = {
+ sources: [
+ {
+ id: 1,
+ name: 'default',
+ tenant_id: 'tenant',
+ config: { host: 'db.internal', name: 'main-db', port: 5432, username: 'etl_user' },
+ },
+ ],
+}
+
+const publicationTables = [
+ { id: 101, schema: 'public', name: 'orders' },
+ { id: 202, schema: 'billing', name: 'invoices' },
+]
+
+const renderDialog = (
+ tableSyncCopy: { type: 'include_tables'; table_ids: number[] } | { type: 'skip_all_tables' }
+) => {
+ addAPIMock({
+ method: 'get',
+ path: '/platform/replication/:ref/sources',
+ response: () => HttpResponse.json<ReplicationSourcesResponse>(mockSources),
+ })
+ addAPIMock({
+ method: 'get',
+ path: '/platform/replication/:ref/sources/:source_id/publications/:publication_name/cost-estimate',
+ response: () =>
+ HttpResponse.json<CostEstimateResponse>({
+ currency: 'usd',
+ pipeline: { hourly_cost: 0.05, monthly_cost: 36.5 },
+ streaming: { rate_per_gb: 3 },
+ table_copy: {
+ rate_per_gb: 0.6,
+ total_bytes: 10_000_000_000,
+ total_cost: 10,
+ tables: costEstimateTables,
+ },
+ }),
+ })
+
+ return customRender(
+ <PipelineCostDialog
+ open
+ isConfirming={false}
+ publicationName="analytics"
+ publicationTables={publicationTables}
+ tableSyncCopy={tableSyncCopy}
+ onOpenChange={vi.fn()}
+ onConfirm={vi.fn()}
+ />
+ )
+}
+
+describe('PipelineCostDialog', () => {
+ beforeEach(() => {
+ costEstimateTables = tables
+ })
+
+ it('shows only the initial-copy tables selected by the policy', async () => {
+ renderDialog({ type: 'include_tables', table_ids: [101] })
+
+ expect(await screen.findByText('public.orders')).toBeInTheDocument()
+ expect(screen.queryByText('billing.invoices')).not.toBeInTheDocument()
+ expect(screen.getAllByText('$0.60')).toHaveLength(3)
+ expect(screen.queryByText('$10.00')).not.toBeInTheDocument()
+ })
+
+ it('shows a zero initial-copy charge while retaining ongoing rates', async () => {
+ renderDialog({ type: 'skip_all_tables' })
+
+ expect(await screen.findByText(/No tables will be initially copied/)).toBeInTheDocument()
+ expect(screen.getByText('$0.00')).toBeInTheDocument()
+ expect(screen.getByText('$0.05/hour')).toBeInTheDocument()
+ expect(screen.getByText('$3.00/GB')).toBeInTheDocument()
+ expect(screen.queryByText('public.orders')).not.toBeInTheDocument()
+ })
+
+ it('does not show a partial total when a selected table estimate is missing', async () => {
+ costEstimateTables = [tables[0]]
+
+ renderDialog({ type: 'include_tables', table_ids: [101, 202] })
+
+ expect(
+ await screen.findByText(/estimate is unavailable for one or more selected tables/)
+ ).toBeInTheDocument()
+ expect(screen.getByText('Unavailable')).toBeInTheDocument()
+ expect(screen.queryByText('$0.60')).not.toBeInTheDocument()
+ })
+})
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PipelineCostDialog.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PipelineCostDialog.tsx
index a7cfe11ea98..11ae2c194db 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PipelineCostDialog.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PipelineCostDialog.tsx
@@ -1,4 +1,5 @@
-import { useEffect } from 'react'
+import { useParams } from 'common'
+import { useEffect, useMemo } from 'react'
import {
Button,
Card,
@@ -20,7 +21,14 @@ import {
} from 'ui'
import { GenericSkeletonLoader } from 'ui-patterns/ShimmeringLoader'
+import {
+ getTableCopyTargets,
+ summarizeTableCopyEstimate,
+ type ReplicationTableIdentity,
+ type TableSyncCopyConfig,
+} from '@/components/interfaces/Database/Replication/TableSyncCopy.utils'
import { useReplicationCostEstimateQuery } from '@/data/replication/cost-estimate-query'
+import { useReplicationSourceId } from '@/data/replication/sources-query'
import { useLatest } from '@/hooks/misc/useLatest'
import { formatBytes, formatCurrency } from '@/lib/helpers'
@@ -29,9 +37,9 @@ const MAX_VISIBLE_TABLES = 10
interface PipelineCostDialogProps {
open: boolean
isConfirming: boolean
- projectRef?: string
- sourceId?: number
publicationName?: string
+ publicationTables: ReplicationTableIdentity[]
+ tableSyncCopy?: TableSyncCopyConfig
onOpenChange: (open: boolean) => void
onConfirm: () => void
}
@@ -46,12 +54,15 @@ interface PipelineCostDialogProps {
export const PipelineCostDialog = ({
open,
isConfirming,
- projectRef,
- sourceId,
publicationName,
+ publicationTables,
+ tableSyncCopy,
onOpenChange,
onConfirm,
}: PipelineCostDialogProps) => {
+ const { ref: projectRef } = useParams()
+ const sourceId = useReplicationSourceId({ projectRef })
+
const onConfirmRef = useLatest(onConfirm)
const {
@@ -61,13 +72,25 @@ export const PipelineCostDialog = ({
isSuccess,
} = useReplicationCostEstimateQuery({ projectRef, sourceId, publicationName }, { enabled: open })
- const tables = estimate?.table_copy.tables ?? []
- const tableCount = tables.length
+ const copyTargets = useMemo(
+ () => getTableCopyTargets(publicationTables, tableSyncCopy),
+ [publicationTables, tableSyncCopy]
+ )
+ const copyEstimate = useMemo(
+ () =>
+ estimate === undefined
+ ? undefined
+ : summarizeTableCopyEstimate(estimate.table_copy.tables, copyTargets),
+ [copyTargets, estimate]
+ )
+ const tables = copyEstimate?.tables ?? []
+ const tableCount = publicationTables.length
+ const copyTableCount = copyTargets.length
const visibleTables = tables.slice(0, MAX_VISIBLE_TABLES)
- const hiddenTableCount = tableCount - visibleTables.length
- const hasRowFilteredTables = tables.some((table) => table.is_row_filtered)
+ const hiddenTableCount = tables.length - visibleTables.length
+ const hasRowFilteredTables = copyEstimate?.hasRowFilteredTables ?? false
- const initialSyncTotal = estimate?.table_copy.total_cost ?? 0
+ const initialSyncTotal = copyEstimate?.estimatedCost ?? 0
useEffect(() => {
if (open && isError) onConfirmRef.current()
@@ -116,7 +139,12 @@ export const PipelineCostDialog = ({
<div className="flex flex-col gap-y-2">
<p className="text-sm font-medium text-foreground">Initial sync</p>
- {tableCount > 0 ? (
+ {copyTableCount === 0 ? (
+ <p className="text-sm text-foreground-light">
+ No tables will be initially copied. All publication tables will still stream
+ new changes, with no initial-copy charge.
+ </p>
+ ) : copyEstimate?.isComplete ? (
<Card>
<Table>
<TableHeader className="[&_th]:h-auto [&_th]:py-2">
@@ -130,7 +158,7 @@ export const PipelineCostDialog = ({
</TableHeader>
<TableBody className="[&_td]:py-2">
{visibleTables.map((table) => (
- <TableRow key={`${table.schema}.${table.name}`}>
+ <TableRow key={JSON.stringify([table.schema, table.name])}>
<TableCell className="font-mono text-xs" translate="no">
{table.schema}.{table.name}
</TableCell>
@@ -155,10 +183,10 @@ export const PipelineCostDialog = ({
<TableRow>
<TableCell className="py-2">Total</TableCell>
<TableCell className="text-right py-2 text-xs">
- {formatBytes(estimate.table_copy.total_bytes)}
+ {formatBytes(copyEstimate.estimatedBytes)}
</TableCell>
<TableCell className="text-right py-2 font-mono" translate="no">
- {formatCurrency(estimate.table_copy.total_cost)}
+ {formatCurrency(copyEstimate.estimatedCost)}
</TableCell>
</TableRow>
</TableFooter>
@@ -166,7 +194,8 @@ export const PipelineCostDialog = ({
</Card>
) : (
<p className="text-sm text-foreground-light">
- This publication has no tables to copy.
+ An initial-copy estimate is unavailable for one or more selected tables. You
+ can still create the pipeline.
</p>
)}
</div>
@@ -211,12 +240,12 @@ export const PipelineCostDialog = ({
className="shrink-0 text-right font-mono text-lg font-semibold text-foreground"
translate="no"
>
- {formatCurrency(initialSyncTotal)}
- {hasRowFilteredTables ? '*' : null}
+ {copyEstimate?.isComplete ? formatCurrency(initialSyncTotal) : 'Unavailable'}
+ {copyEstimate?.isComplete && hasRowFilteredTables ? '*' : null}
</span>
</div>
- {hasRowFilteredTables && (
+ {copyEstimate?.isComplete && hasRowFilteredTables && (
<p className="text-xs text-foreground-lighter">
*Tables with row filters may cost less than shown.
</p>
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx
index 14b853877ab..2d3f65b37f5 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx
@@ -8,22 +8,22 @@ import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
import type { DestinationPanelSchemaType } from './DestinationForm.schema'
import { PublicationsComboBox } from './PublicationsComboBox'
import { useReplicationPublicationsQuery } from '@/data/replication/publications-query'
+import { useReplicationSourceId } from '@/data/replication/sources-query'
type PublicationSelectionProps = {
form: UseFormReturn<DestinationPanelSchemaType>
- sourceId?: number
- visible: boolean
onSelectNewPublication: () => void
}
export const PublicationSelection = ({
form,
- sourceId,
onSelectNewPublication,
}: PublicationSelectionProps) => {
const { ref: projectRef } = useParams()
const { publicationName } = form.watch()
+ const sourceId = useReplicationSourceId({ projectRef })
+
const { data: publications, isSuccess: isSuccessPublications } = useReplicationPublicationsQuery({
projectRef,
sourceId,
@@ -45,7 +45,18 @@ export const PublicationSelection = ({
>
<FormControl>
<PublicationsComboBox
- field={field}
+ field={{
+ ...field,
+ onChange: (value) => {
+ if (value !== field.value) {
+ form.setValue('tableSyncCopyTableIds', [], {
+ shouldDirty: true,
+ shouldValidate: true,
+ })
+ }
+ field.onChange(value)
+ },
+ }}
sourceId={sourceId}
onNewPublicationClick={() => onSelectNewPublication()}
/>
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/TableCopySelection.test.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/TableCopySelection.test.tsx
new file mode 100644
index 00000000000..ce98e6a7b86
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/TableCopySelection.test.tsx
@@ -0,0 +1,162 @@
+import { screen } from '@testing-library/react'
+import type { components } from 'api-types'
+import { HttpResponse } from 'msw'
+import { useForm } from 'react-hook-form'
+import { Form } from 'ui'
+import { describe, expect, it } from 'vitest'
+
+import type { DestinationPanelSchemaType } from './DestinationForm.schema'
+import { TableCopySelection } from './TableCopySelection'
+import { customRender } from '@/tests/lib/custom-render'
+import { addAPIMock, type APIErrorBody } from '@/tests/lib/msw'
+
+type ReplicationSourcesResponse = components['schemas']['ReplicationSourcesResponse']
+type ReplicationPublicationsResponse = components['schemas']['ReplicationPublicationsResponse']
+
+const mockSources: ReplicationSourcesResponse = {
+ sources: [
+ {
+ id: 1,
+ name: 'default',
+ tenant_id: 'tenant',
+ config: { host: 'db.internal', name: 'main-db', port: 5432, username: 'etl_user' },
+ },
+ ],
+}
+
+const mockPublications: ReplicationPublicationsResponse = {
+ publications: [
+ {
+ name: 'analytics',
+ tables: [
+ { id: 101, schema: 'public', name: 'orders' },
+ { id: 202, schema: 'billing', name: 'invoices' },
+ ],
+ },
+ ],
+}
+
+const mockSourcesEndpoint = () => {
+ addAPIMock({
+ method: 'get',
+ path: '/platform/replication/:ref/sources',
+ response: () => HttpResponse.json<ReplicationSourcesResponse>(mockSources),
+ })
+}
+
+const mockPublicationsSuccess = () => {
+ addAPIMock({
+ method: 'get',
+ path: '/platform/replication/:ref/sources/:source_id/publications',
+ response: () => HttpResponse.json<ReplicationPublicationsResponse>(mockPublications),
+ })
+}
+
+const mockPublicationsError = () => {
+ addAPIMock({
+ method: 'get',
+ path: '/platform/replication/:ref/sources/:source_id/publications',
+ response: () => HttpResponse.json<APIErrorBody>({ message: 'Boom' }, { status: 500 }),
+ })
+}
+
+const mockPublicationsPending = () => {
+ addAPIMock({
+ method: 'get',
+ path: '/platform/replication/:ref/sources/:source_id/publications',
+ response: () => new Promise<never>(() => {}),
+ })
+}
+
+const TableCopySelectionHarness = ({
+ editMode,
+ mode = 'include_all_tables',
+ selectedTableIds = [],
+}: {
+ editMode: boolean
+ mode?: DestinationPanelSchemaType['tableSyncCopyMode']
+ selectedTableIds?: string[]
+}) => {
+ const form = useForm<DestinationPanelSchemaType>({
+ defaultValues: {
+ name: 'Analytics',
+ publicationName: 'analytics',
+ tableSyncCopyMode: mode,
+ tableSyncCopyTableIds: selectedTableIds,
+ },
+ })
+
+ return (
+ <Form {...form}>
+ <TableCopySelection form={form} editMode={editMode} />
+ </Form>
+ )
+}
+
+describe('TableCopySelection', () => {
+ it('explains that editing the policy does not recopy synchronized tables', async () => {
+ mockSourcesEndpoint()
+ mockPublicationsSuccess()
+
+ customRender(<TableCopySelectionHarness editMode />)
+
+ expect(
+ await screen.findByText(/will not be re-copied until they require one again/)
+ ).toBeInTheDocument()
+ })
+
+ it('does not show the edit-mode explanation while creating a pipeline', async () => {
+ mockSourcesEndpoint()
+ mockPublicationsSuccess()
+
+ customRender(<TableCopySelectionHarness editMode={false} />)
+
+ await screen.findByText('Copy all tables')
+ expect(
+ screen.queryByText(/will not be re-copied until they require one again/)
+ ).not.toBeInTheDocument()
+ })
+
+ it('summarizes selective copy choices against the publication table count', async () => {
+ mockSourcesEndpoint()
+ mockPublicationsSuccess()
+
+ customRender(
+ <TableCopySelectionHarness
+ editMode={false}
+ mode="include_tables"
+ selectedTableIds={['101']}
+ />
+ )
+
+ expect(
+ await screen.findByText('1 of 2 publication tables will have their existing rows copied.')
+ ).toBeInTheDocument()
+ })
+
+ it('blocks selection and explains when publication tables cannot be loaded', async () => {
+ mockSourcesEndpoint()
+ mockPublicationsError()
+
+ customRender(
+ <TableCopySelectionHarness editMode mode="include_tables" selectedTableIds={['101']} />
+ )
+
+ expect(await screen.findByText(/Publication tables could not be loaded/)).toBeInTheDocument()
+ expect(screen.queryByText(/previously selected table/)).not.toBeInTheDocument()
+ })
+
+ it('disables selective table choices while publication tables are loading', async () => {
+ mockSourcesEndpoint()
+ mockPublicationsPending()
+
+ customRender(
+ <TableCopySelectionHarness editMode mode="include_tables" selectedTableIds={['101', '999']} />
+ )
+
+ const loadingLabel = await screen.findByText('Loading publication tables...')
+ expect(loadingLabel).toBeInTheDocument()
+ expect(loadingLabel.closest('button')).toBeDisabled()
+ expect(screen.queryByText(/previously selected table/)).not.toBeInTheDocument()
+ })
+})
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/TableCopySelection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/TableCopySelection.tsx
new file mode 100644
index 00000000000..0bb08815cd7
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/TableCopySelection.tsx
@@ -0,0 +1,204 @@
+import { useParams } from 'common'
+import { useMemo } from 'react'
+import type { UseFormReturn } from 'react-hook-form'
+import { FormControl, FormField, Select, SelectContent, SelectItem, SelectTrigger } from 'ui'
+import { Admonition } from 'ui-patterns/admonition'
+import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
+import { MultiSelector } from 'ui-patterns/multi-select'
+
+import type { DestinationPanelSchemaType } from './DestinationForm.schema'
+import { getPublicationTableIds } from './DestinationForm.utils'
+import { useReplicationPublicationsQuery } from '@/data/replication/publications-query'
+import { useReplicationSourceId } from '@/data/replication/sources-query'
+
+interface TableCopySelectionProps {
+ form: UseFormReturn<DestinationPanelSchemaType>
+ editMode: boolean
+}
+
+const TABLE_COPY_LABELS = {
+ include_all_tables: 'Copy all tables',
+ skip_all_tables: 'Skip all table copies',
+ include_tables: 'Copy only selected tables',
+ skip_tables: 'Skip selected table copies',
+}
+
+const isSelectiveMode = (mode: DestinationPanelSchemaType['tableSyncCopyMode']) =>
+ mode === 'include_tables' || mode === 'skip_tables'
+
+const tableLabel = ({ schema, name }: { schema: string; name: string }) => `${schema}.${name}`
+
+export const TableCopySelection = ({ form, editMode }: TableCopySelectionProps) => {
+ const { ref: projectRef } = useParams()
+ const sourceId = useReplicationSourceId({ projectRef })
+
+ const { publicationName, tableSyncCopyMode, tableSyncCopyTableIds } = form.watch()
+
+ const {
+ data: publications = [],
+ isPending: isLoadingPublications,
+ isError: isErrorPublications,
+ } = useReplicationPublicationsQuery({ projectRef, sourceId })
+
+ // Only publication tables are ever selectable or displayed by name. A table
+ // id is never resolved outside of the publication response — there's no
+ // other source for a name to come from without a separate, source-wide
+ // catalog fetch, which selective table-copy doesn't need.
+ const publicationTables = useMemo(() => {
+ const publication = publications.find(({ name }) => name === publicationName)
+ return [...(publication?.tables ?? [])].sort((a, b) =>
+ tableLabel(a).localeCompare(tableLabel(b))
+ )
+ }, [publicationName, publications])
+
+ const publicationTableIds = useMemo(
+ () => getPublicationTableIds(publications, publicationName),
+ [publications, publicationName]
+ )
+ const tableLabelsById = new Map(
+ publicationTables.map((table) => [String(table.id), tableLabel(table)])
+ )
+ const selectedPublicationCount = tableSyncCopyTableIds.filter((id) =>
+ publicationTableIds.has(id)
+ ).length
+ const staleSelectedCount =
+ isLoadingPublications || isErrorPublications
+ ? 0
+ : tableSyncCopyTableIds.filter((id) => !publicationTableIds.has(id)).length
+ const tableCount = publicationTables.length
+
+ return (
+ <div className="flex flex-col gap-y-4">
+ <FormField
+ control={form.control}
+ name="tableSyncCopyMode"
+ render={({ field }) => (
+ <FormItemLayout
+ layout="horizontal"
+ label="Initial table copy"
+ description="Choose which existing rows to copy before change streaming begins. All publication tables continue streaming new changes."
+ >
+ <FormControl>
+ <Select value={field.value} onValueChange={field.onChange}>
+ <SelectTrigger>{TABLE_COPY_LABELS[field.value]}</SelectTrigger>
+ <SelectContent>
+ <SelectItem value="include_all_tables" className="[&>span]:top-2.5">
+ <p>Copy all tables</p>
+ <p className="text-foreground-lighter">Copy existing rows from every table.</p>
+ </SelectItem>
+ <SelectItem value="skip_tables" className="[&>span]:top-2.5">
+ <p>Skip selected table copies</p>
+ <p className="text-foreground-lighter">
+ Stream selected tables without copying existing rows.
+ </p>
+ </SelectItem>
+ <SelectItem value="include_tables" className="[&>span]:top-2.5">
+ <p>Copy only selected tables</p>
+ <p className="text-foreground-lighter">
+ Only copy existing rows from selected tables.
+ </p>
+ </SelectItem>
+ <SelectItem value="skip_all_tables" className="[&>span]:top-2.5">
+ <p>Skip all table copies</p>
+ <p className="text-foreground-lighter">
+ Start streaming without copying any existing rows.
+ </p>
+ </SelectItem>
+ </SelectContent>
+ </Select>
+ </FormControl>
+ </FormItemLayout>
+ )}
+ />
+
+ {editMode && (
+ <Admonition type="note" title="Only applies to tables pending initial sync">
+ <p className="leading-normal!">
+ Tables that already completed their initial sync will not be re-copied until they
+ require one again.
+ </p>
+ </Admonition>
+ )}
+
+ {isErrorPublications && (
+ <Admonition type="warning" title="Publication tables could not be loaded">
+ <p className="leading-normal!">
+ Refresh before changing or saving the initial copy configuration.
+ </p>
+ </Admonition>
+ )}
+
+ {isSelectiveMode(tableSyncCopyMode) && (
+ <FormField
+ control={form.control}
+ name="tableSyncCopyTableIds"
+ render={({ field }) => (
+ <FormItemLayout
+ layout="horizontal"
+ label={tableSyncCopyMode === 'skip_tables' ? 'Tables to skip' : 'Tables to copy'}
+ description={
+ tableSyncCopyMode === 'skip_tables'
+ ? `${selectedPublicationCount} of ${tableCount} publication tables will stream without an initial copy.`
+ : `${selectedPublicationCount} of ${tableCount} publication tables will have their existing rows copied.`
+ }
+ >
+ <FormControl>
+ <MultiSelector
+ values={field.value}
+ onValuesChange={(x) => {
+ field.onChange(x)
+ }}
+ disabled={
+ isLoadingPublications ||
+ isErrorPublications ||
+ !publicationName ||
+ tableCount === 0
+ }
+ >
+ <MultiSelector.Trigger
+ badgeLimit={3}
+ renderValue={(id) => tableLabelsById.get(id) ?? `Table ${id}`}
+ label={
+ isLoadingPublications
+ ? 'Loading publication tables...'
+ : publicationName
+ ? 'Select tables...'
+ : 'Select a publication first'
+ }
+ />
+ <MultiSelector.Content>
+ <MultiSelector.List>
+ {publicationTables.map((table) => (
+ <MultiSelector.Item key={table.id} value={String(table.id)}>
+ {tableLabel(table)}
+ </MultiSelector.Item>
+ ))}
+ </MultiSelector.List>
+ </MultiSelector.Content>
+ </MultiSelector>
+ </FormControl>
+
+ {staleSelectedCount > 0 && (
+ <Admonition
+ type="warning"
+ className="mt-2"
+ title={`${
+ staleSelectedCount === 1
+ ? 'A previously selected table is'
+ : `${staleSelectedCount} previously selected tables are`
+ }{' '}
+ no longer in this publication.`}
+ >
+ <p className="leading-normal!">
+ {staleSelectedCount === 1 ? 'It' : 'They'} will be excluded from this pipeline's
+ initial-copy configuration when you save.
+ </p>
+ </Admonition>
+ )}
+ </FormItemLayout>
+ )}
+ />
+ )}
+ </div>
+ )
+}
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ValidationFailuresSection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ValidationFailuresSection.tsx
index 23712bca42f..6ff0e0a4d75 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ValidationFailuresSection.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ValidationFailuresSection.tsx
@@ -34,12 +34,12 @@ export const ValidationFailuresSection = ({
<Admonition
type={hasCriticalFailures ? 'warning' : 'default'}
className="px-5 rounded-none border-0"
- title="Destination configuration issues"
+ title="Configuration issues"
>
<p className="text-sm text-foreground-light mb-2!">
{hasCriticalFailures
? `Please fix all required issues below${hasWarnings ? ' and review the others' : ''} before continuing.`
- : 'The following issues were identified, although you may still create the pipeline and start replication to the destination.'}
+ : 'The following issues were identified. You may still continue after reviewing them.'}
</p>
<p className="text-sm text-foreground-light mb-2!">
Pipeline options are under <strong>Advanced settings</strong> above. Source database
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ValidationWarningsDialog.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ValidationWarningsDialog.tsx
index 51cf36f9663..201c8cba410 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ValidationWarningsDialog.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ValidationWarningsDialog.tsx
@@ -25,6 +25,7 @@ export const ValidationWarningsDialog = ({
onConfirm,
}: ValidationWarningsDialogProps) => {
const hasWarnings = warningCount > 0
+ const action = 'Create and start pipeline'
return (
<AlertDialog open={open} onOpenChange={onOpenChange}>
@@ -32,8 +33,8 @@ export const ValidationWarningsDialog = ({
<AlertDialogHeader>
<AlertDialogTitle>
{hasWarnings
- ? `Create and start pipeline with ${warningCount} ${warningCount === 1 ? 'warning' : 'warnings'}?`
- : 'Create and start pipeline?'}
+ ? `${action} with ${warningCount} ${warningCount === 1 ? 'warning' : 'warnings'}?`
+ : `${action}?`}
</AlertDialogTitle>
<AlertDialogDescription>
{hasWarnings
@@ -49,7 +50,7 @@ export const ValidationWarningsDialog = ({
loading={isLoading}
onClick={onConfirm}
>
- {hasWarnings ? 'Create and start pipeline anyway' : 'Create and start pipeline'}
+ {hasWarnings ? `${action} anyway` : action}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.test.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.test.tsx
new file mode 100644
index 00000000000..8a2c7caf74e
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.test.tsx
@@ -0,0 +1,235 @@
+import { fireEvent, render, screen, waitFor } from '@testing-library/react'
+import type { ButtonHTMLAttributes, PropsWithChildren } from 'react'
+import type { UseFormReturn } from 'react-hook-form'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+import type { DestinationPanelSchemaType } from './DestinationForm.schema'
+import { DestinationForm } from './index'
+
+const mocks = vi.hoisted(() => ({
+ refetchPublications: vi.fn(),
+ resetValidation: vi.fn(),
+ submitPipeline: vi.fn(),
+ validateConfiguration: vi.fn(),
+}))
+
+const existingBatch = {
+ max_fill_ms: 500,
+ max_bytes: 8_388_608,
+ memory_budget_ratio: 0.2,
+}
+
+const existingDestination = {
+ destinationId: 7,
+ pipelineId: 8,
+ enabled: true,
+ statusName: 'started',
+}
+
+const sourcesData = { sources: [{ id: 42, name: 'project-ref' }] }
+const publications = [
+ {
+ name: 'analytics',
+ tables: [{ id: 101, schema: 'public', name: 'orders' }],
+ },
+]
+const destinationData = {
+ name: 'Analytics',
+ config: {
+ big_query: {
+ project_id: 'example-project',
+ dataset_id: 'analytics',
+ connection_pool_size: 5,
+ },
+ },
+}
+const pipelineData = {
+ config: {
+ publication_name: 'analytics',
+ table_sync_copy: { type: 'include_tables' as const, table_ids: [101, 999] },
+ batch: existingBatch,
+ max_table_sync_workers: 4,
+ max_copy_connections_per_table: 1,
+ },
+}
+const projectSettings = { region: 'ap-southeast-1' }
+
+vi.mock('common', () => ({ useParams: () => ({ ref: 'project-ref' }) }))
+
+vi.mock('framer-motion', () => ({
+ AnimatePresence: ({ children }: PropsWithChildren) => children,
+ motion: { div: ({ children }: PropsWithChildren) => <div>{children}</div> },
+}))
+
+vi.mock('ui', () => ({
+ Button: ({
+ children,
+ loading: _loading,
+ ...props
+ }: ButtonHTMLAttributes<HTMLButtonElement> & { loading?: boolean }) => (
+ <button tabIndex={0} {...props}>
+ {children}
+ </button>
+ ),
+ DialogSectionSeparator: () => null,
+ Form: ({ children }: PropsWithChildren) => children,
+ Select: ({ children }: PropsWithChildren) => <div>{children}</div>,
+ SelectContent: ({ children }: PropsWithChildren) => <div>{children}</div>,
+ SelectItem: ({ children }: PropsWithChildren) => <div>{children}</div>,
+ SelectTrigger: ({ children }: PropsWithChildren) => <div>{children}</div>,
+ SelectValue: () => null,
+ SheetFooter: ({ children }: PropsWithChildren) => <div>{children}</div>,
+ SheetSection: ({ children }: PropsWithChildren) => <div>{children}</div>,
+ Tooltip: ({ children }: PropsWithChildren) => <div>{children}</div>,
+ TooltipContent: ({ children }: PropsWithChildren) => <div>{children}</div>,
+ TooltipTrigger: ({ children }: PropsWithChildren) => <span>{children}</span>,
+}))
+
+vi.mock('ui-patterns/admonition', () => ({
+ Admonition: ({ children }: PropsWithChildren) => <div>{children}</div>,
+}))
+
+vi.mock('ui-patterns/form/FormItemLayout/FormItemLayout', () => ({
+ FormItemLayout: ({ children }: PropsWithChildren) => <div>{children}</div>,
+}))
+
+vi.mock('../../useIsETLPrivateAlpha', () => ({
+ useIsETLBigQueryPrivateAlpha: () => true,
+ useIsETLClickHousePrivateAlpha: () => false,
+ useIsETLDucklakePrivateAlpha: () => false,
+ useIsETLIcebergPrivateAlpha: () => false,
+ useIsETLSnowflakePrivateAlpha: () => false,
+}))
+
+vi.mock('./useDestinationForm', () => ({
+ useDestinationForm: () => ({
+ isValidating: false,
+ validateConfiguration: mocks.validateConfiguration,
+ isSaving: false,
+ submitPipeline: mocks.submitPipeline,
+ hasRunValidation: false,
+ destinationValidationFailures: [],
+ pipelineValidationFailures: [],
+ resetValidation: mocks.resetValidation,
+ }),
+}))
+
+vi.mock('@/hooks/misc/useCheckPermissions', () => ({
+ useAsyncCheckPermissions: () => ({ can: false }),
+}))
+
+vi.mock('@/data/api-keys/api-keys-query', () => ({ useAPIKeys: () => ({ data: undefined }) }))
+
+vi.mock('@/data/config/project-settings-v2-query', () => ({
+ useProjectSettingsV2Query: () => ({ data: projectSettings }),
+}))
+
+vi.mock('@/data/replication/sources-query', () => ({
+ useReplicationSourcesQuery: () => ({ data: sourcesData }),
+ useReplicationSourceId: () => sourcesData.sources[0]?.id,
+}))
+
+vi.mock('@/data/replication/publications-query', () => ({
+ useReplicationPublicationsQuery: () => ({
+ data: publications,
+ isPending: false,
+ isError: false,
+ isSuccess: true,
+ refetch: mocks.refetchPublications,
+ }),
+}))
+
+vi.mock('@/data/replication/destination-by-id-query', () => ({
+ useReplicationDestinationByIdQuery: () => ({
+ data: destinationData,
+ isError: false,
+ isSuccess: true,
+ }),
+}))
+
+vi.mock('@/data/replication/pipeline-by-id-query', () => ({
+ useReplicationPipelineByIdQuery: () => ({
+ data: pipelineData,
+ isError: false,
+ isSuccess: true,
+ }),
+}))
+
+vi.mock('./DestinationNameInput', () => ({ DestinationNameInput: () => null }))
+vi.mock('./PublicationSelection', () => ({ PublicationSelection: () => null }))
+vi.mock('./TableCopySelection', () => ({
+ TableCopySelection: ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => (
+ <div>{form.formState.errors.tableSyncCopyTableIds?.message}</div>
+ ),
+}))
+vi.mock('./AdvancedSettings', () => ({ AdvancedSettings: () => null }))
+vi.mock('./BigQuery/Fields', () => ({ BigQueryFields: () => null }))
+vi.mock('./AnalyticsBucket/Fields', () => ({ AnalyticsBucketFields: () => null }))
+vi.mock('./DuckLake/Fields', () => ({ DuckLakeFields: () => null }))
+vi.mock('./Snowflake/Fields', () => ({ SnowflakeFields: () => null }))
+vi.mock('./ClickHouse/Fields', () => ({ ClickHouseFields: () => null }))
+vi.mock('./NewPublicationPanel', () => ({ NewPublicationPanel: () => null }))
+vi.mock('./NoDestinationsAvailable', () => ({ NoDestinationsAvailable: () => null }))
+vi.mock('./PipelineCostDialog', () => ({ PipelineCostDialog: () => null }))
+vi.mock('./ValidationFailuresSection', () => ({ ValidationFailuresSection: () => null }))
+vi.mock('./ValidationWarningsDialog', () => ({ ValidationWarningsDialog: () => null }))
+vi.mock('@/components/interfaces/Storage/AnalyticsBuckets/CreateAnalyticsBucketSheet', () => ({
+ CreateAnalyticsBucketSheet: () => null,
+}))
+
+describe('DestinationForm edit submission', () => {
+ beforeEach(() => {
+ vi.clearAllMocks()
+ pipelineData.config.table_sync_copy.table_ids = [101, 999]
+ mocks.submitPipeline.mockResolvedValue(undefined)
+ mocks.validateConfiguration.mockResolvedValue({ canContinue: true, warnings: [] })
+ })
+
+ it('bypasses create validation and submits the pruned table policy with the existing batch', async () => {
+ const onClose = vi.fn()
+
+ render(
+ <DestinationForm
+ selectedType="BigQuery"
+ visible
+ existingDestination={existingDestination}
+ onClose={onClose}
+ />
+ )
+
+ fireEvent.click(screen.getByRole('button', { name: 'Apply and restart pipeline' }))
+
+ await waitFor(() => expect(mocks.submitPipeline).toHaveBeenCalledOnce())
+
+ expect(mocks.validateConfiguration).not.toHaveBeenCalled()
+ expect(mocks.submitPipeline).toHaveBeenCalledWith({
+ data: expect.objectContaining({
+ tableSyncCopyMode: 'include_tables',
+ tableSyncCopyTableIds: ['101'],
+ }),
+ existingDestination,
+ existingBatch,
+ onSuccess: expect.any(Function),
+ onClose,
+ })
+ })
+
+ it('rejects an edit when every selected table has left the publication', async () => {
+ pipelineData.config.table_sync_copy.table_ids = [999]
+
+ render(
+ <DestinationForm
+ selectedType="BigQuery"
+ visible
+ existingDestination={existingDestination}
+ onClose={vi.fn()}
+ />
+ )
+
+ fireEvent.click(screen.getByRole('button', { name: 'Apply and restart pipeline' }))
+
+ expect(await screen.findByText('Select at least one table')).toBeInTheDocument()
+ expect(mocks.validateConfiguration).not.toHaveBeenCalled()
+ expect(mocks.submitPipeline).not.toHaveBeenCalled()
+ })
+})
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx
index cd7cc1c9b93..03780cc6076 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx
@@ -6,6 +6,7 @@ import { Loader2 } from 'lucide-react'
import { useEffect, useMemo, useRef, useState } from 'react'
import { useForm } from 'react-hook-form'
import { AWS_REGIONS } from 'shared-data'
+import { toast } from 'sonner'
import {
Button,
DialogSectionSeparator,
@@ -21,6 +22,7 @@ import {
TooltipContent,
TooltipTrigger,
} from 'ui'
+import { Admonition } from 'ui-patterns/admonition'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
import * as z from 'zod'
@@ -40,7 +42,12 @@ import { BigQueryFields } from './BigQuery/Fields'
import { getClickHouseValidationIssues } from './ClickHouse/ClickHouse.utils'
import { ClickHouseFields } from './ClickHouse/Fields'
import { DestinationPanelFormSchema as FormSchema } from './DestinationForm.schema'
-import { areValidationFailuresEqual, generateDefaultValues } from './DestinationForm.utils'
+import {
+ areValidationFailuresEqual,
+ buildTableSyncCopyConfig,
+ generateDefaultValues,
+ pruneStaleSelectedTableIds,
+} from './DestinationForm.utils'
import { DestinationNameInput } from './DestinationNameInput'
import { getDucklakeValidationIssues } from './DuckLake/DuckLake.utils'
import { DuckLakeFields } from './DuckLake/Fields'
@@ -50,6 +57,7 @@ import { PipelineCostDialog } from './PipelineCostDialog'
import { PublicationSelection } from './PublicationSelection'
import { SnowflakeFields } from './Snowflake/Fields'
import { getSnowflakeValidationIssues } from './Snowflake/Snowflake.utils'
+import { TableCopySelection } from './TableCopySelection'
import { useDestinationForm } from './useDestinationForm'
import { ValidationFailuresSection } from './ValidationFailuresSection'
import { ValidationWarningsDialog } from './ValidationWarningsDialog'
@@ -60,7 +68,7 @@ import { useProjectSettingsV2Query } from '@/data/config/project-settings-v2-que
import { useReplicationDestinationByIdQuery } from '@/data/replication/destination-by-id-query'
import { useReplicationPipelineByIdQuery } from '@/data/replication/pipeline-by-id-query'
import { useReplicationPublicationsQuery } from '@/data/replication/publications-query'
-import { useReplicationSourcesQuery } from '@/data/replication/sources-query'
+import { useReplicationSourceId } from '@/data/replication/sources-query'
import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions'
import { BASE_PATH, IS_STAGING_OR_LOCAL } from '@/lib/constants'
@@ -123,8 +131,7 @@ export const DestinationForm = ({
])
const hasNoAvailableDestinations = availableDestinations.length === 0
- const { data: sourcesData } = useReplicationSourcesQuery({ projectRef })
- const sourceId = sourcesData?.sources.find((s) => s.name === projectRef)?.id
+ const sourceId = useReplicationSourceId({ projectRef })
const {
data: publications = [],
@@ -132,19 +139,32 @@ export const DestinationForm = ({
refetch: refetchPublications,
} = useReplicationPublicationsQuery({ projectRef, sourceId })
- const { data: destinationData } = useReplicationDestinationByIdQuery({
+ const {
+ data: destinationData,
+ isError: isErrorDestination,
+ isSuccess: isSuccessDestination,
+ } = useReplicationDestinationByIdQuery({
projectRef,
destinationId: existingDestination?.destinationId,
})
- const { data: pipelineData } = useReplicationPipelineByIdQuery({
+ const {
+ data: pipelineData,
+ isError: isErrorPipeline,
+ isSuccess: isSuccessPipeline,
+ } = useReplicationPipelineByIdQuery({
projectRef,
pipelineId: existingDestination?.pipelineId,
})
+ const isErrorExistingConfig = editMode && (isErrorDestination || isErrorPipeline)
+ const isExistingConfigReady = !editMode || (isSuccessDestination && isSuccessPipeline)
+ // Revealed API keys are only ever consumed as the default catalog token for
+ // Analytics Bucket (Iceberg) destinations, so don't fetch them for other
+ // destination types.
const { data: apiKeysData } = useAPIKeys(
{ projectRef, reveal: true },
- { enabled: canReadAPIKeys }
+ { enabled: canReadAPIKeys && selectedType === 'Analytics Bucket' && !editMode }
)
const { serviceKey } = apiKeysData ?? {}
const catalogToken = serviceKey?.api_key ?? ''
@@ -160,9 +180,7 @@ export const DestinationForm = ({
destinationValidationFailures,
pipelineValidationFailures,
resetValidation,
- } = useDestinationForm({
- selectedType,
- })
+ } = useDestinationForm({ selectedType })
const defaultValues = useMemo(
() =>
@@ -190,6 +208,22 @@ export const DestinationForm = ({
})
}
+ const selectedPublicationTableIds = pruneStaleSelectedTableIds({
+ mode: data.tableSyncCopyMode,
+ selectedTableIds: data.tableSyncCopyTableIds,
+ publications,
+ publicationName: data.publicationName,
+ })
+
+ if (
+ isSuccessPublications &&
+ (data.tableSyncCopyMode === 'include_tables' ||
+ data.tableSyncCopyMode === 'skip_tables') &&
+ selectedPublicationTableIds.length === 0
+ ) {
+ addRequiredFieldError('tableSyncCopyTableIds', 'Select at least one table')
+ }
+
if (selectedType === 'BigQuery') {
getBigQueryValidationIssues(data, { secretsOptional: editMode }).forEach(
({ path, message }) => {
@@ -235,8 +269,28 @@ export const DestinationForm = ({
const hasValidationFailures = allValidationFailures.some((f) => f.failure_type === 'critical')
const validationWarnings = allValidationFailures.filter((f) => f.failure_type === 'warning')
+ const pendingTableSyncCopy = useMemo(
+ () =>
+ pendingFormValues === null
+ ? undefined
+ : buildTableSyncCopyConfig({
+ mode: pendingFormValues.tableSyncCopyMode,
+ selectedTableIds: pendingFormValues.tableSyncCopyTableIds,
+ }),
+ [pendingFormValues]
+ )
+ const pendingPublicationTables = useMemo(
+ () =>
+ publications.find(({ name }) => name === pendingFormValues?.publicationName)?.tables ?? [],
+ [pendingFormValues?.publicationName, publications]
+ )
+
const isSubmitDisabled =
- isSaving || isSelectedPublicationMissing || (!editMode && hasNoAvailableDestinations)
+ isSaving ||
+ !isExistingConfigReady ||
+ !isSuccessPublications ||
+ isSelectedPublicationMissing ||
+ (!editMode && hasNoAvailableDestinations)
const getSubmitButtonText = () => {
if (editMode) {
@@ -259,11 +313,32 @@ export const DestinationForm = ({
setShowCostDialog(true)
}
- const onSubmit = async (data: z.infer<typeof FormSchema>) => {
+ const onSubmit = async (rawData: z.infer<typeof FormSchema>) => {
+ if (!isSuccessPublications) {
+ toast.error('Publication tables are unavailable. Refresh and try again.')
+ return
+ }
+
+ // Drop any previously selected id that has since fallen out of the
+ // publication before validating a create or submitting an edit.
+ const data: z.infer<typeof FormSchema> = {
+ ...rawData,
+ tableSyncCopyTableIds: pruneStaleSelectedTableIds({
+ mode: rawData.tableSyncCopyMode,
+ selectedTableIds: rawData.tableSyncCopyTableIds,
+ publications,
+ publicationName: rawData.publicationName,
+ }),
+ }
+
+ // Pipeline prerequisite validation models a new pipeline and cannot
+ // account for resources already owned by an existing pipeline. Edits keep
+ // the established direct-update flow after pruning stale table ids.
if (editMode) {
await submitPipeline({
data,
existingDestination,
+ existingBatch: pipelineData?.config.batch,
onSuccess: () => form.reset(defaultValues),
onClose,
})
@@ -309,8 +384,7 @@ export const DestinationForm = ({
openCostDialog(data)
}
- // Confirming the warnings advances to the cost dialog rather than submitting directly, so the
- // cost estimate is always the last thing shown before the pipeline starts.
+ // Confirming create warnings advances to the cost dialog.
const handleValidationWarningsConfirm = () => {
if (!pendingFormValues) return
setShowValidationWarningsDialog(false)
@@ -326,6 +400,7 @@ export const DestinationForm = ({
await submitPipeline({
data: values,
existingDestination,
+ existingBatch: pipelineData?.config.batch,
onSuccess: () => form.reset(defaultValues),
onClose,
})
@@ -352,103 +427,112 @@ export const DestinationForm = ({
) : (
<Form {...form}>
<form id={formId} onSubmit={form.handleSubmit(onSubmit)}>
- <div className="p-5 flex flex-col gap-y-6">
- <p className="text-sm font-medium text-foreground">Destination details</p>
+ <fieldset disabled={!isExistingConfigReady} className="contents">
+ <div className="p-5 flex flex-col gap-y-6">
+ {isErrorExistingConfig && (
+ <Admonition type="warning">
+ <p className="leading-normal!">
+ The existing destination or pipeline settings could not be loaded. Refresh
+ before applying changes.
+ </p>
+ </Admonition>
+ )}
+ <p className="text-sm font-medium text-foreground">Destination details</p>
+
+ <div className="flex flex-col gap-y-4">
+ <DestinationNameInput form={form} />
+ <PublicationSelection
+ form={form}
+ onSelectNewPublication={() => setPublicationPanelVisible(true)}
+ />
+ <TableCopySelection form={form} editMode={editMode} />
+ <FormItemLayout
+ isReactForm={false}
+ layout="horizontal"
+ label="Region"
+ description={
+ <span className="text-foreground-lighter">
+ Pipelines run in{' '}
+ <Tooltip>
+ <TooltipTrigger className={InlineLinkClassName}>
+ {PIPELINE_REGION.displayName}
+ </TooltipTrigger>
+ <TooltipContent side="bottom">{PIPELINE_REGION.code}</TooltipContent>
+ </Tooltip>
+ . In your destination provider, choose the closest available region.
+ </span>
+ }
+ >
+ <Select disabled value={PIPELINE_REGION.code}>
+ <SelectTrigger>
+ <SelectValue placeholder="Select a region" />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value={PIPELINE_REGION.code}>
+ <div className="flex gap-x-3 items-center">
+ <img
+ alt="region icon"
+ className="w-5 rounded-xs"
+ src={`${BASE_PATH}/img/regions/${PIPELINE_REGION.code}.svg`}
+ />
+ <p className="flex items-center gap-x-2">
+ <span>{PIPELINE_REGION.displayName}</span>
+ <span className="text-xs text-foreground-lighter font-mono">
+ {PIPELINE_REGION.code}
+ </span>
+ </p>
+ </div>
+ </SelectItem>
+ </SelectContent>
+ </Select>
+ </FormItemLayout>
+ </div>
+ </div>
- <div className="flex flex-col gap-y-4">
- <DestinationNameInput form={form} />
- <PublicationSelection
+ <DialogSectionSeparator />
+
+ {selectedType === 'BigQuery' && etlEnableBigQuery ? (
+ <BigQueryFields form={form} editMode={editMode} />
+ ) : selectedType === 'Analytics Bucket' && etlEnableIceberg ? (
+ <AnalyticsBucketFields
form={form}
- sourceId={sourceId}
- visible={visible}
- onSelectNewPublication={() => setPublicationPanelVisible(true)}
+ editMode={editMode}
+ onSelectNewBucket={() => setNewBucketSheetVisible(true)}
/>
- <FormItemLayout
- isReactForm={false}
- layout="horizontal"
- label="Region"
- description={
- <span className="text-foreground-lighter">
- Pipelines run in{' '}
- <Tooltip>
- <TooltipTrigger className={InlineLinkClassName}>
- {PIPELINE_REGION.displayName}
- </TooltipTrigger>
- <TooltipContent side="bottom">{PIPELINE_REGION.code}</TooltipContent>
- </Tooltip>
- . In your destination provider, choose the closest available region.
- </span>
+ ) : selectedType === 'DuckLake' && etlEnableDucklake ? (
+ <DuckLakeFields form={form} editMode={editMode} />
+ ) : selectedType === 'Snowflake' && etlEnableSnowflake ? (
+ <SnowflakeFields
+ form={form}
+ editMode={editMode}
+ hasStoredPrivateKeyPassphrase={
+ editMode && !!defaultValues.snowflakePrivateKeyPassphrase
}
- >
- <Select disabled value={PIPELINE_REGION.code}>
- <SelectTrigger>
- <SelectValue placeholder="Select a region" />
- </SelectTrigger>
- <SelectContent>
- <SelectItem value={PIPELINE_REGION.code}>
- <div className="flex gap-x-3 items-center">
- <img
- alt="region icon"
- className="w-5 rounded-xs"
- src={`${BASE_PATH}/img/regions/${PIPELINE_REGION.code}.svg`}
- />
- <p className="flex items-center gap-x-2">
- <span>{PIPELINE_REGION.displayName}</span>
- <span className="text-xs text-foreground-lighter font-mono">
- {PIPELINE_REGION.code}
- </span>
- </p>
- </div>
- </SelectItem>
- </SelectContent>
- </Select>
- </FormItemLayout>
- </div>
- </div>
-
- <DialogSectionSeparator />
-
- {selectedType === 'BigQuery' && etlEnableBigQuery ? (
- <BigQueryFields form={form} editMode={editMode} />
- ) : selectedType === 'Analytics Bucket' && etlEnableIceberg ? (
- <AnalyticsBucketFields
- form={form}
- editMode={editMode}
- onSelectNewBucket={() => setNewBucketSheetVisible(true)}
- />
- ) : selectedType === 'DuckLake' && etlEnableDucklake ? (
- <DuckLakeFields form={form} editMode={editMode} />
- ) : selectedType === 'Snowflake' && etlEnableSnowflake ? (
- <SnowflakeFields
- form={form}
- editMode={editMode}
- hasStoredPrivateKeyPassphrase={
- editMode && !!defaultValues.snowflakePrivateKeyPassphrase
- }
- />
- ) : selectedType === 'ClickHouse' && etlEnableClickHouse ? (
- <ClickHouseFields
- form={form}
- hasStoredPassword={editMode && !!defaultValues.clickhousePassword}
- />
- ) : null}
-
- <DialogSectionSeparator />
-
- <AdvancedSettings type={selectedType} form={form} />
-
- {!editMode && hasRunValidation && !isValidating && (
- <>
- <DialogSectionSeparator />
-
- <div ref={validationSectionRef}>
- <ValidationFailuresSection
- destinationFailures={destinationValidationFailures}
- pipelineFailures={pipelineValidationFailures}
- />
- </div>
- </>
- )}
+ />
+ ) : selectedType === 'ClickHouse' && etlEnableClickHouse ? (
+ <ClickHouseFields
+ form={form}
+ hasStoredPassword={editMode && !!defaultValues.clickhousePassword}
+ />
+ ) : null}
+
+ <DialogSectionSeparator />
+
+ <AdvancedSettings type={selectedType} form={form} />
+
+ {!editMode && hasRunValidation && !isValidating && (
+ <>
+ <DialogSectionSeparator />
+
+ <div ref={validationSectionRef}>
+ <ValidationFailuresSection
+ destinationFailures={destinationValidationFailures}
+ pipelineFailures={pipelineValidationFailures}
+ />
+ </div>
+ </>
+ )}
+ </fieldset>
</form>
</Form>
)}
@@ -490,10 +574,18 @@ export const DestinationForm = ({
</SheetFooter>
<NewPublicationPanel
- sourceId={sourceId}
visible={publicationPanelVisible}
onClose={(newPublication?: string) => {
- if (newPublication) form.setValue('publicationName', newPublication)
+ if (newPublication) {
+ form.setValue('tableSyncCopyTableIds', [], {
+ shouldDirty: true,
+ shouldValidate: true,
+ })
+ form.setValue('publicationName', newPublication, {
+ shouldDirty: true,
+ shouldValidate: true,
+ })
+ }
setPublicationPanelVisible(false)
}}
/>
@@ -514,9 +606,9 @@ export const DestinationForm = ({
<PipelineCostDialog
open={showCostDialog}
isConfirming={isSaving}
- projectRef={projectRef}
- sourceId={sourceId}
publicationName={pendingFormValues?.publicationName}
+ publicationTables={pendingPublicationTables}
+ tableSyncCopy={pendingTableSyncCopy}
onOpenChange={setShowCostDialog}
onConfirm={handleCostConfirm}
/>
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/useDestinationForm.test.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/useDestinationForm.test.tsx
new file mode 100644
index 00000000000..ea4832617bb
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/useDestinationForm.test.tsx
@@ -0,0 +1,205 @@
+import { act, renderHook } from '@testing-library/react'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+import type { DestinationPanelSchemaType } from './DestinationForm.schema'
+import { useDestinationForm } from './useDestinationForm'
+
+const mocks = vi.hoisted(() => ({
+ validateDestination: vi.fn(),
+ validatePipeline: vi.fn(),
+ createS3AccessKey: vi.fn(),
+ createNamespace: vi.fn(),
+ createDestinationPipeline: vi.fn(),
+ updateDestinationPipeline: vi.fn(),
+ startPipeline: vi.fn(),
+ setRequestStatus: vi.fn(),
+}))
+
+vi.mock('common', () => ({ useParams: () => ({ ref: 'project-ref' }) }))
+vi.mock('@/data/replication/sources-query', () => ({
+ useReplicationSourcesQuery: () => ({
+ data: { sources: [{ id: 42, name: 'project-ref' }] },
+ }),
+}))
+vi.mock('@/data/replication/validate-destination-mutation', () => ({
+ useValidateDestinationMutation: () => ({
+ mutateAsync: mocks.validateDestination,
+ isPending: false,
+ }),
+}))
+vi.mock('@/data/replication/validate-pipeline-mutation', () => ({
+ useValidatePipelineMutation: () => ({
+ mutateAsync: mocks.validatePipeline,
+ isPending: false,
+ }),
+}))
+vi.mock('@/data/storage/s3-access-key-create-mutation', () => ({
+ useS3AccessKeyCreateMutation: () => ({
+ mutateAsync: mocks.createS3AccessKey,
+ isPending: false,
+ }),
+}))
+vi.mock('@/data/storage/iceberg-namespace-create-mutation', () => ({
+ useIcebergNamespaceCreateMutation: () => ({
+ mutateAsync: mocks.createNamespace,
+ isPending: false,
+ }),
+}))
+vi.mock('@/data/replication/create-destination-pipeline-mutation', () => ({
+ useCreateDestinationPipelineMutation: () => ({
+ mutateAsync: mocks.createDestinationPipeline,
+ isPending: false,
+ }),
+}))
+vi.mock('@/data/replication/update-destination-pipeline-mutation', () => ({
+ useUpdateDestinationPipelineMutation: () => ({
+ mutateAsync: mocks.updateDestinationPipeline,
+ isPending: false,
+ }),
+}))
+vi.mock('@/data/replication/start-pipeline-mutation', () => ({
+ useStartPipelineMutation: () => ({
+ mutateAsync: mocks.startPipeline,
+ isPending: false,
+ }),
+}))
+vi.mock('@/state/replication-pipeline-request-status', () => ({
+ PipelineStatusRequestStatus: {
+ RestartRequested: 'restart-requested',
+ StartRequested: 'start-requested',
+ },
+ usePipelineRequestStatus: () => ({ setRequestStatus: mocks.setRequestStatus }),
+}))
+
+const formData = {
+ name: 'Analytics',
+ publicationName: 'analytics',
+ tableSyncCopyMode: 'include_tables',
+ tableSyncCopyTableIds: ['101'],
+ maxFillMs: 500,
+ projectId: 'example-project',
+ datasetId: 'analytics',
+ serviceAccountKey: '',
+ connectionPoolSize: 5,
+} as DestinationPanelSchemaType
+
+describe('useDestinationForm validation', () => {
+ beforeEach(() => {
+ vi.clearAllMocks()
+ mocks.validateDestination.mockResolvedValue({ validation_failures: [] })
+ mocks.validatePipeline.mockResolvedValue({ validation_failures: [] })
+ })
+
+ it('validates both destination and pipeline while creating', async () => {
+ const { result } = renderHook(() => useDestinationForm({ selectedType: 'BigQuery' }))
+
+ await act(async () => {
+ await result.current.validateConfiguration({
+ data: { ...formData, serviceAccountKey: '{"type":"service_account"}' },
+ onValidationFail: vi.fn(),
+ })
+ })
+
+ expect(mocks.validateDestination).toHaveBeenCalledOnce()
+ expect(mocks.validatePipeline).toHaveBeenCalledWith(
+ expect.objectContaining({
+ projectRef: 'project-ref',
+ sourceId: 42,
+ publicationName: 'analytics',
+ tableSyncCopy: { type: 'include_tables', table_ids: [101] },
+ })
+ )
+ })
+
+ it('blocks creation when pipeline validation returns a critical failure', async () => {
+ const failure = {
+ failure_type: 'critical',
+ name: 'Invalid table selection',
+ reason: 'Refresh the publication selection.',
+ }
+ mocks.validatePipeline.mockResolvedValue({ validation_failures: [failure] })
+ const onValidationFail = vi.fn()
+ const { result } = renderHook(() => useDestinationForm({ selectedType: 'BigQuery' }))
+
+ let validationResult: Awaited<ReturnType<typeof result.current.validateConfiguration>>
+ await act(async () => {
+ validationResult = await result.current.validateConfiguration({
+ data: { ...formData, serviceAccountKey: '{"type":"service_account"}' },
+ onValidationFail,
+ })
+ })
+
+ expect(validationResult!).toEqual({ canContinue: false, warnings: [] })
+ expect(onValidationFail).toHaveBeenCalledOnce()
+ })
+
+ it('preserves hidden batch fields and submits the selected table-copy policy on edit', async () => {
+ const { result } = renderHook(() => useDestinationForm({ selectedType: 'BigQuery' }))
+
+ await act(async () => {
+ await result.current.submitPipeline({
+ data: formData,
+ existingDestination: {
+ destinationId: 7,
+ pipelineId: 8,
+ enabled: true,
+ statusName: 'started',
+ },
+ existingBatch: {
+ max_fill_ms: 200,
+ max_bytes: 8_388_608,
+ memory_budget_ratio: 0.2,
+ },
+ onSuccess: vi.fn(),
+ onClose: vi.fn(),
+ })
+ })
+
+ expect(mocks.updateDestinationPipeline).toHaveBeenCalledWith(
+ expect.objectContaining({
+ destinationId: 7,
+ pipelineId: 8,
+ pipelineConfig: expect.objectContaining({
+ tableSyncCopy: { type: 'include_tables', table_ids: [101] },
+ batch: {
+ maxFillMs: 500,
+ maxBytes: 8_388_608,
+ memoryBudgetRatio: 0.2,
+ },
+ }),
+ }),
+ expect.any(Object)
+ )
+ expect(mocks.createDestinationPipeline).not.toHaveBeenCalled()
+ expect(mocks.startPipeline).not.toHaveBeenCalled()
+ })
+
+ it('omits an unchanged batch when editing only the table-copy policy', async () => {
+ const { result } = renderHook(() => useDestinationForm({ selectedType: 'BigQuery' }))
+
+ await act(async () => {
+ await result.current.submitPipeline({
+ data: formData,
+ existingDestination: {
+ destinationId: 7,
+ pipelineId: 8,
+ enabled: true,
+ statusName: 'started',
+ },
+ existingBatch: {
+ max_fill_ms: formData.maxFillMs,
+ max_bytes: 0,
+ memory_budget_ratio: 2,
+ },
+ onSuccess: vi.fn(),
+ onClose: vi.fn(),
+ })
+ })
+
+ const updateParams = mocks.updateDestinationPipeline.mock.calls[0][0]
+ expect(updateParams.pipelineConfig).toMatchObject({
+ tableSyncCopy: { type: 'include_tables', table_ids: [101] },
+ })
+ expect(updateParams.pipelineConfig).not.toHaveProperty('batch')
+ })
+})
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/useDestinationForm.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/useDestinationForm.ts
index 6f69c79eddb..b317f4e5f98 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/useDestinationForm.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/useDestinationForm.ts
@@ -4,16 +4,19 @@ import { toast } from 'sonner'
import * as z from 'zod'
import { type DestinationType, type ExistingDestination } from '../DestinationPanel.types'
-import { CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
+import { CREATE_NEW_NAMESPACE, DEFAULT_MAX_FILL_MS } from './DestinationForm.constants'
import { DestinationPanelFormSchema as FormSchema } from './DestinationForm.schema'
import {
+ buildBatchConfig,
buildDestinationConfig,
buildDestinationConfigForValidation,
+ buildTableSyncCopyConfig,
} from './DestinationForm.utils'
import {
useCreateDestinationPipelineMutation,
type BatchConfig,
} from '@/data/replication/create-destination-pipeline-mutation'
+import type { ReplicationPipelineByIdData } from '@/data/replication/pipeline-by-id-query'
import { useReplicationSourcesQuery } from '@/data/replication/sources-query'
import { useStartPipelineMutation } from '@/data/replication/start-pipeline-mutation'
import { useUpdateDestinationPipelineMutation } from '@/data/replication/update-destination-pipeline-mutation'
@@ -111,6 +114,17 @@ export const useDestinationForm = ({ selectedType }: { selectedType: Destination
}) => {
if (!projectRef || !sourceId) return { canContinue: false, warnings: [] }
+ let tableSyncCopy
+ try {
+ tableSyncCopy = buildTableSyncCopyConfig({
+ mode: data.tableSyncCopyMode,
+ selectedTableIds: data.tableSyncCopyTableIds,
+ })
+ } catch (error) {
+ toast.error((error as Error).message)
+ return { canContinue: false, warnings: [] }
+ }
+
setHasRunValidation(true)
// Call both validation endpoints in parallel and wait for both to complete
@@ -118,13 +132,18 @@ export const useDestinationForm = ({ selectedType }: { selectedType: Destination
const results = await Promise.allSettled([
validateDestination({
projectRef,
- destinationConfig: buildDestinationConfigForValidation({ projectRef, selectedType, data }),
+ destinationConfig: buildDestinationConfigForValidation({
+ projectRef,
+ selectedType,
+ data,
+ }),
sourceId,
publicationName: data.publicationName,
maxFillMs: data.maxFillMs,
maxTableSyncWorkers: data.maxTableSyncWorkers,
maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,
invalidatedSlotBehavior: data.invalidatedSlotBehavior,
+ tableSyncCopy,
}),
validatePipeline({
projectRef,
@@ -134,6 +153,7 @@ export const useDestinationForm = ({ selectedType }: { selectedType: Destination
maxTableSyncWorkers: data.maxTableSyncWorkers,
maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,
invalidatedSlotBehavior: data.invalidatedSlotBehavior,
+ tableSyncCopy,
}),
])
@@ -180,11 +200,13 @@ export const useDestinationForm = ({ selectedType }: { selectedType: Destination
const submitPipeline = async ({
data,
existingDestination,
+ existingBatch,
onSuccess,
onClose,
}: {
data: z.infer<typeof FormSchema>
existingDestination?: ExistingDestination
+ existingBatch?: ReplicationPipelineByIdData['config']['batch']
onSuccess: () => void
onClose: () => void
}) => {
@@ -204,15 +226,26 @@ export const useDestinationForm = ({ selectedType }: { selectedType: Destination
if (!destinationConfig) throw new Error('Destination configuration is missing')
- const batchConfig: BatchConfig | undefined =
- data.maxFillMs !== undefined ? { maxFillMs: data.maxFillMs } : undefined
+ const shouldSendBatch =
+ !editMode || data.maxFillMs !== (existingBatch?.max_fill_ms ?? DEFAULT_MAX_FILL_MS)
+ const batchConfig: BatchConfig | undefined = shouldSendBatch
+ ? buildBatchConfig({
+ maxFillMs: data.maxFillMs,
+ existingBatch,
+ })
+ : undefined
const hasBatchFields = batchConfig !== undefined
+ const tableSyncCopy = buildTableSyncCopyConfig({
+ mode: data.tableSyncCopyMode,
+ selectedTableIds: data.tableSyncCopyTableIds,
+ })
const pipelineConfig = {
publicationName: data.publicationName,
maxTableSyncWorkers: data.maxTableSyncWorkers,
maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,
invalidatedSlotBehavior: data.invalidatedSlotBehavior,
+ tableSyncCopy,
...(hasBatchFields ? { batch: batchConfig } : {}),
}
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.test.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.test.tsx
index fea7170f381..7d62e31f8c6 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.test.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.test.tsx
@@ -155,7 +155,7 @@ describe('DestinationTypeSelection', () => {
big_query: {
project_id: 'gcp-proj',
dataset_id: 'analytics',
- service_account_key: '{}',
+ connection_pool_size: 5,
},
},
}),
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationRow.test.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationRow.test.tsx
index d3e34656cd2..f0c81baf936 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationRow.test.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationRow.test.tsx
@@ -72,7 +72,11 @@ const addDestinationMock = () =>
id: DESTINATION_ID,
name: 'My BigQuery Destination',
config: {
- big_query: { project_id: 'gcp-proj', dataset_id: 'analytics', service_account_key: '{}' },
+ big_query: {
+ project_id: 'gcp-proj',
+ dataset_id: 'analytics',
+ connection_pool_size: 5,
+ },
},
}),
})
@@ -92,7 +96,10 @@ const addPipelinesMock = () =>
destination_id: DESTINATION_ID,
destination_name: 'My BigQuery Destination',
replicator_id: 9001,
- config: { publication_name: 'supabase_realtime' },
+ config: {
+ publication_name: 'supabase_realtime',
+ table_sync_copy: { type: 'include_all_tables' },
+ },
},
],
}),
@@ -227,6 +234,9 @@ describe('DestinationRow', () => {
addPipelineStatusMock('started')
addReplicationStatusMock(0, [
{
+ id: 1,
+ schema: 'public',
+ name: 'orders',
table_id: 1,
table_name: 'public.orders',
state: { name: 'error', reason: 'table not found', retry_policy: { policy: 'no_retry' } },
@@ -249,6 +259,9 @@ describe('DestinationRow', () => {
addPipelineStatusMock('stopped')
addReplicationStatusMock(0, [
{
+ id: 1,
+ schema: 'public',
+ name: 'orders',
table_id: 1,
table_name: 'public.orders',
state: { name: 'error', reason: 'table not found', retry_policy: { policy: 'no_retry' } },
diff --git a/apps/studio/components/interfaces/Database/Replication/ErroredTableDetails.tsx b/apps/studio/components/interfaces/Database/Replication/ErroredTableDetails.tsx
index 2a56d42d8c7..1bc5ff7cdd3 100644
--- a/apps/studio/components/interfaces/Database/Replication/ErroredTableDetails.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/ErroredTableDetails.tsx
@@ -14,7 +14,7 @@ export const ErroredTableDetails = ({ table }: ErroredTableDetailsProps) => {
const { ref: projectRef } = useParams()
const state = table.state as Extract<ReplicationPipelineTableStatus['state'], { name: 'error' }>
- const tableName = table.table_name
+ const tableName = `${table.schema}.${table.name}`
const retryPolicy = state.retry_policy.policy
if (!isValidRetryPolicy(state.retry_policy)) {
diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx
index 0423fe8272a..8da3285a784 100644
--- a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx
@@ -81,8 +81,9 @@ export const ReplicationPipelineStatus = () => {
} | null>(null)
const [showRestartDialog, setShowRestartDialog] = useState(false)
const [selectedTableForRestart, setSelectedTableForRestart] = useState<{
- tableId: number
- tableName: string
+ id: number
+ schema: string
+ name: string
} | null>(null)
const [showBatchRestartDialog, setShowBatchRestartDialog] = useState(false)
const [batchRestartMode, setBatchRestartMode] = useState<'all' | 'errored' | null>(null)
@@ -143,11 +144,11 @@ export const ReplicationPipelineStatus = () => {
const displayState = getPipelineDisplayState(requestStatus, statusName)
const config = getDisabledStateConfig({ requestStatus, statusName })
- // Sort tables by name for consistent ordering (memoized)
+ // Sort tables by schema and name for consistent ordering (memoized)
const tableStatuses = useMemo(
() =>
- (replicationStatusData?.table_statuses || []).sort((a, b) =>
- a.table_name.localeCompare(b.table_name)
+ (replicationStatusData?.table_statuses || []).sort(
+ (a, b) => a.schema.localeCompare(b.schema) || a.name.localeCompare(b.name)
),
[replicationStatusData?.table_statuses]
)
@@ -160,7 +161,7 @@ export const ReplicationPipelineStatus = () => {
searchString.length === 0
? tableStatuses
: tableStatuses.filter((table) =>
- table.table_name.toLowerCase().includes(searchString.toLowerCase())
+ `${table.schema}.${table.name}`.toLowerCase().includes(searchString.toLowerCase())
),
[tableStatuses, searchString]
)
@@ -171,13 +172,7 @@ export const ReplicationPipelineStatus = () => {
)
const erroredTables = useMemo(
- () =>
- tableStatuses.filter(
- (table) =>
- table.state.name === 'error' &&
- 'retry_policy' in table.state &&
- table.state.retry_policy?.policy === 'manual_retry'
- ),
+ () => tableStatuses.filter((table) => table.state.name === 'error'),
[tableStatuses]
)
@@ -382,9 +377,9 @@ export const ReplicationPipelineStatus = () => {
<div className="rounded-sm border border-default/50 bg-surface-200/40">
<ul className="divide-y divide-default/40">
{tablesWithLag.map((table) => (
- <li key={`${table.table_id}-${table.table_name}`} className="px-3 py-2">
+ <li key={table.id} className="px-3 py-2">
<SlotLagMetricsInline
- tableName={table.table_name}
+ tableName={`${table.schema}.${table.name}`}
metrics={table.table_sync_lag as SlotLagMetrics}
/>
</li>
@@ -454,7 +449,7 @@ export const ReplicationPipelineStatus = () => {
tooltip={{
content: {
side: 'left',
- text: !hasErroredTables ? 'No tables require manual retry' : undefined,
+ text: !hasErroredTables ? 'No failed tables' : undefined,
},
}}
>
@@ -485,7 +480,7 @@ export const ReplicationPipelineStatus = () => {
</TableHeader>
<TableBody>
{filteredTableStatuses.map((table) => {
- const isRestarting = restartingTableIds.has(table.table_id)
+ const isRestarting = restartingTableIds.has(table.id)
const isErrorState = table.state.name === 'error'
const errorReason =
isErrorState && 'reason' in table.state ? table.state.reason : undefined
@@ -493,7 +488,7 @@ export const ReplicationPipelineStatus = () => {
isErrorState && 'solution' in table.state ? table.state.solution : undefined
return (
<TableReplicationRow
- key={table.table_id}
+ key={table.id}
table={table}
isRestarting={isRestarting}
showDisabledState={showDisabledState}
@@ -502,8 +497,9 @@ export const ReplicationPipelineStatus = () => {
isPipelineStopped={statusName === PipelineStatusName.STOPPED}
onSelectRestart={() => {
setSelectedTableForRestart({
- tableId: table.table_id,
- tableName: table.table_name,
+ id: table.id,
+ schema: table.schema,
+ name: table.name,
})
setShowRestartDialog(true)
}}
@@ -511,7 +507,7 @@ export const ReplicationPipelineStatus = () => {
isErrorState && errorReason
? () => {
setSelectedTableError({
- tableName: table.table_name,
+ tableName: `${table.schema}.${table.name}`,
reason: errorReason,
solution: errorSolution,
})
@@ -581,18 +577,18 @@ export const ReplicationPipelineStatus = () => {
<RestartTableDialog
open={showRestartDialog}
onOpenChange={setShowRestartDialog}
- tableId={selectedTableForRestart.tableId}
- tableName={selectedTableForRestart.tableName}
+ table={selectedTableForRestart}
+ tableSyncCopy={pipeline?.config.table_sync_copy}
sourceId={pipeline?.source_id}
publicationName={pipeline?.config.publication_name}
pipelineStatusName={statusName}
onRestartStart={() => {
- setRestartingTableIds((prev) => new Set(prev).add(selectedTableForRestart.tableId))
+ setRestartingTableIds((prev) => new Set(prev).add(selectedTableForRestart.id))
}}
onRestartComplete={() => {
setRestartingTableIds((prev) => {
const next = new Set(prev)
- next.delete(selectedTableForRestart.tableId)
+ next.delete(selectedTableForRestart.id)
return next
})
}}
@@ -619,6 +615,7 @@ export const ReplicationPipelineStatus = () => {
tables={tableStatuses}
sourceId={pipeline?.source_id}
publicationName={pipeline?.config.publication_name}
+ tableSyncCopy={pipeline?.config.table_sync_copy}
pipelineStatusName={statusName}
onRestartStart={(tableIds) => {
setRestartingTableIds((prev) => new Set([...prev, ...tableIds]))
diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.types.ts b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.types.ts
index 568cadb6125..6b9a80dc8a5 100644
--- a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.types.ts
+++ b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.types.ts
@@ -28,6 +28,12 @@ export type SlotLagMetricKey =
| 'reply_time_lag'
export type TableState = {
+ id: number
+ schema: string
+ name: string
+ // Deprecated compatibility aliases for `id` and `schema`.`name`. Prefer the
+ // structured fields above; the API keeps these only until platform and
+ // etl-api remove them in a follow-up release.
table_id: number
table_name: string
state:
diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/TableReplicationRow.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/TableReplicationRow.tsx
index 8aac3f7cce7..68135368a2b 100644
--- a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/TableReplicationRow.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/TableReplicationRow.tsx
@@ -39,7 +39,9 @@ export const TableReplicationRow = ({
<TableRow>
<TableCell className="align-top">
<div className="flex items-center gap-x-2">
- <p>{table.table_name}</p>
+ <p>
+ {table.schema}.{table.name}
+ </p>
<ButtonTooltip
asChild
@@ -53,7 +55,7 @@ export const TableReplicationRow = ({
<Link
target="_blank"
rel="noopener noreferrer"
- href={`/project/${ref}/editor/${table.table_id}`}
+ href={`/project/${ref}/editor/${table.id}`}
/>
</ButtonTooltip>
</div>
@@ -104,7 +106,7 @@ export const TableReplicationRow = ({
className="w-7"
icon={<RotateCcw />}
disabled={showDisabledState || isRestarting || isAnyRestartInProgress}
- aria-label={`Restart replication for ${table.table_name}`}
+ aria-label={`Restart replication for ${table.schema}.${table.name}`}
onClick={onSelectRestart}
/>
</TooltipTrigger>
diff --git a/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.render.test.tsx b/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.render.test.tsx
new file mode 100644
index 00000000000..a8ded5429e2
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.render.test.tsx
@@ -0,0 +1,33 @@
+import { render, screen } from '@testing-library/react'
+import { describe, expect, it, vi } from 'vitest'
+
+import { RestartCostEstimate } from './RestartCostEstimate'
+
+const mocks = vi.hoisted(() => ({
+ costEstimateQuery: vi.fn(() => ({ data: undefined, isFetching: false })),
+}))
+
+vi.mock('@/data/replication/cost-estimate-query', () => ({
+ useReplicationCostEstimateQuery: mocks.costEstimateQuery,
+}))
+
+describe('RestartCostEstimate', () => {
+ it('does not request an estimate when the table-copy policy skips every target', () => {
+ render(
+ <RestartCostEstimate
+ open
+ projectRef="project-ref"
+ sourceId={42}
+ publicationName="analytics"
+ tables={[]}
+ />
+ )
+
+ expect(screen.getByText('No additional initial-sync copy charge')).toBeInTheDocument()
+ expect(screen.getByText('$0.00')).toBeInTheDocument()
+ expect(mocks.costEstimateQuery).toHaveBeenCalledWith(
+ { projectRef: 'project-ref', sourceId: 42, publicationName: 'analytics' },
+ { enabled: false }
+ )
+ })
+})
diff --git a/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.test.ts b/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.test.ts
index c63198b7cb5..397d907a5d7 100644
--- a/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.test.ts
+++ b/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.test.ts
@@ -28,7 +28,12 @@ const tables = [
describe('calculateRestartCostEstimate', () => {
it('sums only the tables being restarted', () => {
- expect(calculateRestartCostEstimate(tables, ['public.orders', 'public.customers'])).toEqual({
+ expect(
+ calculateRestartCostEstimate(tables, [
+ { schema: 'public', name: 'orders' },
+ { schema: 'public', name: 'customers' },
+ ])
+ ).toMatchObject({
isComplete: true,
estimatedBytes: 3_000_000_000,
estimatedCost: 2,
@@ -38,7 +43,10 @@ describe('calculateRestartCostEstimate', () => {
it('marks the estimate incomplete when a restarted table is missing', () => {
expect(
- calculateRestartCostEstimate(tables, ['public.orders', 'public.missing']).isComplete
+ calculateRestartCostEstimate(tables, [
+ { schema: 'public', name: 'orders' },
+ { schema: 'public', name: 'missing' },
+ ]).isComplete
).toBe(false)
})
})
diff --git a/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.tsx b/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.tsx
index 2558a0b6b9a..618983a6b8f 100644
--- a/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/RestartCostEstimate.tsx
@@ -1,6 +1,11 @@
import { useMemo } from 'react'
import { GenericSkeletonLoader } from 'ui-patterns/ShimmeringLoader'
+import {
+ summarizeTableCopyEstimate,
+ type ReplicationTableIdentity,
+ type TableCopyEstimate,
+} from './TableSyncCopy.utils'
import { useReplicationCostEstimateQuery } from '@/data/replication/cost-estimate-query'
import { formatBytes, formatCurrency } from '@/lib/helpers'
@@ -9,58 +14,55 @@ interface RestartCostEstimateProps {
projectRef?: string
sourceId?: number
publicationName?: string
- tableNames: string[]
+ tables: Pick<ReplicationTableIdentity, 'schema' | 'name'>[]
}
-interface EstimatedTable {
- schema: string
- name: string
- estimated_bytes: number
- estimated_cost: number
- is_row_filtered: boolean
-}
-
-export const calculateRestartCostEstimate = (tables: EstimatedTable[], tableNames: string[]) => {
- const tableNameSet = new Set(tableNames)
- const matchedTables = tables.filter((table) => tableNameSet.has(`${table.schema}.${table.name}`))
-
- return {
- isComplete: matchedTables.length === tableNames.length,
- estimatedBytes: matchedTables.reduce((total, table) => total + table.estimated_bytes, 0),
- estimatedCost: matchedTables.reduce((total, table) => total + table.estimated_cost, 0),
- hasRowFilteredTables: matchedTables.some((table) => table.is_row_filtered),
- }
-}
+export const calculateRestartCostEstimate = (
+ estimates: TableCopyEstimate[],
+ tables: Pick<ReplicationTableIdentity, 'schema' | 'name'>[]
+) => summarizeTableCopyEstimate(estimates, tables)
export const RestartCostEstimate = ({
open,
projectRef,
sourceId,
publicationName,
- tableNames,
+ tables,
}: RestartCostEstimateProps) => {
const { data: estimate, isFetching } = useReplicationCostEstimateQuery(
{ projectRef, sourceId, publicationName },
- { enabled: open }
+ { enabled: open && tables.length > 0 }
)
const restartEstimate = useMemo(
() =>
estimate === undefined
? undefined
- : calculateRestartCostEstimate(estimate.table_copy.tables, tableNames),
- [estimate, tableNames]
+ : calculateRestartCostEstimate(estimate.table_copy.tables, tables),
+ [estimate, tables]
)
return (
<div className="border-t p-4">
- {isFetching ? (
+ {tables.length === 0 ? (
+ <div className="flex items-center justify-between gap-x-6">
+ <div className="min-w-0">
+ <p className="text-sm font-medium">No additional initial-sync copy charge</p>
+ <p className="text-xs text-foreground-lighter">
+ Initial copy is skipped by this pipeline's table-copy policy.
+ </p>
+ </div>
+ <span className="shrink-0 font-mono text-lg font-semibold" translate="no">
+ {formatCurrency(0)}
+ </span>
+ </div>
+ ) : isFetching ? (
<GenericSkeletonLoader className="w-full" />
) : restartEstimate?.isComplete ? (
<div className="flex items-center justify-between gap-x-6">
<div className="min-w-0">
<p className="text-sm font-medium">Estimated additional initial sync</p>
<p className="text-xs text-foreground-lighter">
- {formatBytes(restartEstimate.estimatedBytes)} across {tableNames.length}{' '}
- {tableNames.length === 1 ? 'table' : 'tables'}
+ {formatBytes(restartEstimate.estimatedBytes)} across {tables.length}{' '}
+ {tables.length === 1 ? 'table' : 'tables'}
</p>
</div>
<span className="shrink-0 font-mono text-lg font-semibold" translate="no">
@@ -71,7 +73,7 @@ export const RestartCostEstimate = ({
) : (
<p className="text-xs text-foreground-lighter">
A cost estimate is unavailable. You can still restart the{' '}
- {tableNames.length === 1 ? 'table' : 'tables'}.
+ {tables.length === 1 ? 'table' : 'tables'}.
</p>
)}
{restartEstimate?.isComplete && restartEstimate.hasRowFilteredTables && (
diff --git a/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsx b/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsx
index 17561ae774f..30daeba8b1d 100644
--- a/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsx
@@ -13,13 +13,18 @@ import {
import { PipelineStatusName } from './Replication.constants'
import { RestartCostEstimate } from './RestartCostEstimate'
+import {
+ shouldCopyTable,
+ type ReplicationTableIdentity,
+ type TableSyncCopyConfig,
+} from './TableSyncCopy.utils'
import { useRollbackTablesMutation } from '@/data/replication/rollback-tables-mutation'
interface RestartTableDialogProps {
open: boolean
onOpenChange: (open: boolean) => void
- tableId: number
- tableName: string
+ table: ReplicationTableIdentity
+ tableSyncCopy?: TableSyncCopyConfig
sourceId?: number
publicationName?: string
pipelineStatusName?: PipelineStatusName
@@ -30,8 +35,8 @@ interface RestartTableDialogProps {
export const RestartTableDialog = ({
open,
onOpenChange,
- tableId,
- tableName,
+ table,
+ tableSyncCopy,
sourceId,
publicationName,
pipelineStatusName,
@@ -40,6 +45,8 @@ export const RestartTableDialog = ({
}: RestartTableDialogProps) => {
const { ref: projectRef, pipelineId: _pipelineId } = useParams()
const pipelineId = Number(_pipelineId)
+ const tableName = `${table.schema}.${table.name}`
+ const willCopyTable = shouldCopyTable(tableSyncCopy, table.id)
const { mutate: rollbackTables, isPending: isResetting } = useRollbackTablesMutation({
onSuccess: () => {
@@ -64,7 +71,7 @@ export const RestartTableDialog = ({
rollbackTables({
projectRef,
pipelineId,
- target: { type: 'single_table', table_id: tableId },
+ target: { type: 'single_table', table_id: table.id },
rollbackType: 'full',
pipelineStatusName,
})
@@ -84,11 +91,19 @@ export const RestartTableDialog = ({
<code className="text-code-inline">{tableName}</code> from scratch:
</p>
<ul className="list-disc list-inside space-y-1.5 pl-2">
- <li>
- <strong>The table's initial sync will restart.</strong> All existing data will be
- copied again from the source. This initial sync is billed in addition to previous
- initial syncs.
- </li>
+ {willCopyTable ? (
+ <li>
+ <strong>The table's initial sync will restart.</strong> All existing data will
+ be copied again from the source. This initial sync is billed in addition to
+ previous initial syncs.
+ </li>
+ ) : (
+ <li>
+ <strong>No initial copy will run.</strong> New changes will resume streaming
+ without backfilling existing source rows. There is no additional initial-copy
+ charge.
+ </li>
+ )}
<li>
<strong>Existing downstream data will be deleted.</strong> Any replicated data for
this table will be removed.
@@ -109,7 +124,7 @@ export const RestartTableDialog = ({
projectRef={projectRef}
sourceId={sourceId}
publicationName={publicationName}
- tableNames={[tableName]}
+ tables={willCopyTable ? [table] : []}
/>
<AlertDialogFooter>
<AlertDialogCancel disabled={isResetting}>Cancel</AlertDialogCancel>
diff --git a/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.test.ts b/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.test.ts
new file mode 100644
index 00000000000..023afa18441
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.test.ts
@@ -0,0 +1,82 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+ getTableCopyTargets,
+ shouldCopyTable,
+ summarizeTableCopyEstimate,
+ type TableSyncCopyConfig,
+} from './TableSyncCopy.utils'
+
+const tables = [
+ { id: 101, schema: 'public', name: 'orders' },
+ { id: 202, schema: 'billing', name: 'orders' },
+]
+
+describe('shouldCopyTable', () => {
+ it('defaults an omitted policy to copying every table', () => {
+ expect(shouldCopyTable(undefined, 101)).toBe(true)
+ })
+
+ it.each<[TableSyncCopyConfig, number[]]>([
+ [{ type: 'include_all_tables' }, [101, 202]],
+ [{ type: 'skip_all_tables' }, []],
+ [{ type: 'include_tables', table_ids: [202] }, [202]],
+ [{ type: 'skip_tables', table_ids: [202] }, [101]],
+ ])('filters copy targets for $0', (config, expectedIds) => {
+ expect(getTableCopyTargets(tables, config).map(({ id }) => id)).toEqual(expectedIds)
+ })
+})
+
+describe('summarizeTableCopyEstimate', () => {
+ const estimates = [
+ {
+ schema: 'public',
+ name: 'orders.v2',
+ estimated_bytes: 100,
+ estimated_cost: 1,
+ is_row_filtered: false,
+ },
+ {
+ schema: 'billing',
+ name: 'orders.v2',
+ estimated_bytes: 200,
+ estimated_cost: 2,
+ is_row_filtered: true,
+ },
+ ]
+
+ it('matches schema-qualified identities without dot-delimited collisions', () => {
+ expect(
+ summarizeTableCopyEstimate(estimates, [{ schema: 'billing', name: 'orders.v2' }])
+ ).toMatchObject({
+ isComplete: true,
+ estimatedBytes: 200,
+ estimatedCost: 2,
+ hasRowFilteredTables: true,
+ tables: [estimates[1]],
+ })
+ })
+
+ it('marks an estimate incomplete instead of presenting a partial total', () => {
+ expect(
+ summarizeTableCopyEstimate(estimates, [
+ { schema: 'public', name: 'orders.v2' },
+ { schema: 'missing', name: 'table' },
+ ])
+ ).toMatchObject({
+ isComplete: false,
+ estimatedBytes: 100,
+ estimatedCost: 1,
+ })
+ })
+
+ it('returns a complete zero-cost summary when no table will be copied', () => {
+ expect(summarizeTableCopyEstimate(estimates, [])).toEqual({
+ isComplete: true,
+ tables: [],
+ estimatedBytes: 0,
+ estimatedCost: 0,
+ hasRowFilteredTables: false,
+ })
+ })
+})
diff --git a/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.ts b/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.ts
new file mode 100644
index 00000000000..71fe5b9f0e1
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.ts
@@ -0,0 +1,66 @@
+export type TableSyncCopyConfig =
+ | { type: 'include_all_tables' }
+ | { type: 'skip_all_tables' }
+ | { type: 'include_tables'; table_ids: number[] }
+ | { type: 'skip_tables'; table_ids: number[] }
+
+export type ReplicationTableIdentity = {
+ id: number
+ schema: string
+ name: string
+}
+
+export type TableCopyEstimate = {
+ schema: string
+ name: string
+ estimated_bytes: number
+ estimated_cost: number
+ is_row_filtered: boolean
+}
+
+export const shouldCopyTable = (
+ config: TableSyncCopyConfig | null | undefined,
+ tableId: number
+) => {
+ const resolvedConfig: TableSyncCopyConfig = config ?? { type: 'include_all_tables' }
+
+ switch (resolvedConfig.type) {
+ case 'include_all_tables':
+ return true
+ case 'skip_all_tables':
+ return false
+ case 'include_tables':
+ return resolvedConfig.table_ids.includes(tableId)
+ case 'skip_tables':
+ return !resolvedConfig.table_ids.includes(tableId)
+ }
+}
+
+export const getTableCopyTargets = <T extends { id: number }>(
+ tables: readonly T[],
+ config: TableSyncCopyConfig | null | undefined
+) => tables.filter(({ id }) => shouldCopyTable(config, id))
+
+const getTableIdentityKey = ({ schema, name }: { schema: string; name: string }) =>
+ JSON.stringify([schema, name])
+
+export const summarizeTableCopyEstimate = (
+ estimates: readonly TableCopyEstimate[],
+ targets: readonly Pick<ReplicationTableIdentity, 'schema' | 'name'>[]
+) => {
+ const estimatesByIdentity = new Map(
+ estimates.map((estimate) => [getTableIdentityKey(estimate), estimate])
+ )
+ const matchedTables = targets.flatMap((target) => {
+ const estimate = estimatesByIdentity.get(getTableIdentityKey(target))
+ return estimate === undefined ? [] : [estimate]
+ })
+
+ return {
+ isComplete: matchedTables.length === targets.length,
+ tables: matchedTables,
+ estimatedBytes: matchedTables.reduce((total, table) => total + table.estimated_bytes, 0),
+ estimatedCost: matchedTables.reduce((total, table) => total + table.estimated_cost, 0),
+ hasRowFilteredTables: matchedTables.some((table) => table.is_row_filtered),
+ }
+}
diff --git a/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/NamespaceWithTables/TableRowComponent.tsx b/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/NamespaceWithTables/TableRowComponent.tsx
index a4e936a0d5d..7b68b016af4 100644
--- a/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/NamespaceWithTables/TableRowComponent.tsx
+++ b/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/NamespaceWithTables/TableRowComponent.tsx
@@ -124,9 +124,9 @@ export const TableRowComponent = ({ table, schema, namespace }: TableRowComponen
setIsUpdatingReplication(true)
// [Joshen ALPHA] Assumption here is that all the namespace tables have _changelog as suffix
// May need to update if that assumption falls short (e.g for those dealing with iceberg APIs directly)
- const updatedTables = publication.tables.filter(
- (x) => table.name !== getNamespaceTableNameFromPostgresTableName(x)
- )
+ const updatedTables = publication.tables
+ .filter((x) => table.name !== getNamespaceTableNameFromPostgresTableName(x))
+ .map(({ schema, name }) => ({ schema, name }))
await updatePublication({
projectRef,
sourceId,
@@ -158,9 +158,9 @@ export const TableRowComponent = ({ table, schema, namespace }: TableRowComponen
try {
setIsUpdatingReplication(true)
- const updatedTables = publication.tables.concat([
- { schema: pgTable.schema, name: pgTable.name },
- ])
+ const updatedTables = publication.tables
+ .map(({ schema, name }) => ({ schema, name }))
+ .concat([{ schema: pgTable.schema, name: pgTable.name }])
await updatePublication({
projectRef,
sourceId,
diff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.test.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.test.ts
index 66004344a18..5264a361700 100644
--- a/apps/studio/data/replication/create-destination-pipeline-mutation.test.ts
+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.test.ts
@@ -1,6 +1,31 @@
import { describe, expect, it } from 'vitest'
-import { buildDucklakeApiConfig } from './create-destination-pipeline-mutation'
+import {
+ buildDucklakeApiConfig,
+ buildPipelineApiConfig,
+} from './create-destination-pipeline-mutation'
+
+describe('buildPipelineApiConfig', () => {
+ it('maps selective initial-copy configuration to the ETL API shape', () => {
+ expect(
+ buildPipelineApiConfig({
+ publicationName: 'analytics',
+ batch: { maxFillMs: 500, maxBytes: 8_388_608, memoryBudgetRatio: 0.2 },
+ maxTableSyncWorkers: 4,
+ maxCopyConnectionsPerTable: 2,
+ invalidatedSlotBehavior: 'recreate',
+ tableSyncCopy: { type: 'skip_tables', table_ids: [101, 202] },
+ })
+ ).toEqual({
+ publication_name: 'analytics',
+ batch: { max_fill_ms: 500, max_bytes: 8_388_608, memory_budget_ratio: 0.2 },
+ max_table_sync_workers: 4,
+ max_copy_connections_per_table: 2,
+ invalidated_slot_behavior: 'recreate',
+ table_sync_copy: { type: 'skip_tables', table_ids: [101, 202] },
+ })
+ })
+})
describe('buildDucklakeApiConfig', () => {
it('maps a "Use Supabase" config with catalog-level pool size + metadata schema', () => {
diff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.ts
index 30db38dc6ca..4ada0784bd8 100644
--- a/apps/studio/data/replication/create-destination-pipeline-mutation.ts
+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.ts
@@ -4,9 +4,12 @@ import { toast } from 'sonner'
import { optionalSecret } from './destination-secret-utils'
import { replicationKeys } from './keys'
+import type { TableSyncCopyConfig } from '@/components/interfaces/Database/Replication/TableSyncCopy.utils'
import { handleError, post } from '@/data/fetchers'
import type { ResponseError, UseCustomMutationOptions } from '@/types'
+export type { TableSyncCopyConfig } from '@/components/interfaces/Database/Replication/TableSyncCopy.utils'
+
export type DestinationConfig =
| { bigQuery: BigQueryDestinationConfig }
| { iceberg: IcebergDestinationConfig }
@@ -139,20 +142,47 @@ export type ClickHouseDestinationConfig = {
export type BatchConfig = {
maxFillMs?: number
+ maxBytes?: number
+ memoryBudgetRatio?: number
+}
+
+export type PipelineConfig = {
+ publicationName: string
+ batch?: BatchConfig
+ maxTableSyncWorkers?: number
+ maxCopyConnectionsPerTable?: number
+ invalidatedSlotBehavior?: 'error' | 'recreate'
+ tableSyncCopy: TableSyncCopyConfig
}
+export const buildPipelineApiConfig = ({
+ publicationName,
+ batch,
+ maxTableSyncWorkers,
+ maxCopyConnectionsPerTable,
+ invalidatedSlotBehavior,
+ tableSyncCopy,
+}: PipelineConfig) => ({
+ publication_name: publicationName,
+ max_table_sync_workers: maxTableSyncWorkers,
+ max_copy_connections_per_table: maxCopyConnectionsPerTable,
+ invalidated_slot_behavior: invalidatedSlotBehavior,
+ table_sync_copy: tableSyncCopy,
+ batch: batch
+ ? {
+ max_fill_ms: batch.maxFillMs,
+ max_bytes: batch.maxBytes,
+ memory_budget_ratio: batch.memoryBudgetRatio,
+ }
+ : undefined,
+})
+
export type CreateDestinationPipelineParams = {
projectRef: string
destinationName: string
destinationConfig: DestinationConfig
sourceId: number
- pipelineConfig: {
- publicationName: string
- batch?: BatchConfig
- maxTableSyncWorkers?: number
- maxCopyConnectionsPerTable?: number
- invalidatedSlotBehavior?: 'error' | 'recreate'
- }
+ pipelineConfig: PipelineConfig
}
async function createDestinationPipeline(
@@ -160,13 +190,7 @@ async function createDestinationPipeline(
projectRef,
destinationName: destinationName,
destinationConfig,
- pipelineConfig: {
- publicationName,
- batch,
- maxTableSyncWorkers,
- maxCopyConnectionsPerTable,
- invalidatedSlotBehavior,
- },
+ pipelineConfig,
sourceId,
}: CreateDestinationPipelineParams,
signal?: AbortSignal
@@ -250,13 +274,7 @@ async function createDestinationPipeline(
)
}
- const pipeline_config = {
- publication_name: publicationName,
- max_table_sync_workers: maxTableSyncWorkers,
- max_copy_connections_per_table: maxCopyConnectionsPerTable,
- invalidated_slot_behavior: invalidatedSlotBehavior,
- batch: batch ? { max_fill_ms: batch.maxFillMs } : undefined,
- }
+ const pipeline_config = buildPipelineApiConfig(pipelineConfig)
const { data, error } = await post('/platform/replication/{ref}/destinations-pipelines', {
params: { path: { ref: projectRef } },
diff --git a/apps/studio/data/replication/sources-query.ts b/apps/studio/data/replication/sources-query.ts
index 1be9a53a848..c10c1344a1e 100644
--- a/apps/studio/data/replication/sources-query.ts
+++ b/apps/studio/data/replication/sources-query.ts
@@ -42,3 +42,8 @@ export const useReplicationSourcesQuery = <TData = ReplicationSourcesData>(
retry: checkReplicationFeatureFlagRetry,
...options,
})
+
+export const useReplicationSourceId = ({ projectRef }: { projectRef?: string }) => {
+ const { data: sourcesData } = useReplicationSourcesQuery({ projectRef })
+ return sourcesData?.sources.find((s) => s.name === projectRef)?.id
+}
diff --git a/apps/studio/data/replication/update-destination-pipeline-mutation.ts b/apps/studio/data/replication/update-destination-pipeline-mutation.ts
index 587baa777e4..8356324f701 100644
--- a/apps/studio/data/replication/update-destination-pipeline-mutation.ts
+++ b/apps/studio/data/replication/update-destination-pipeline-mutation.ts
@@ -3,9 +3,10 @@ import type { components } from 'api-types'
import { toast } from 'sonner'
import {
- BatchConfig,
buildDucklakeApiConfig,
+ buildPipelineApiConfig,
DestinationConfig,
+ PipelineConfig,
} from './create-destination-pipeline-mutation'
import { optionalSecret } from './destination-secret-utils'
import { replicationKeys } from './keys'
@@ -19,13 +20,7 @@ export type UpdateDestinationPipelineParams = {
destinationName: string
destinationConfig: DestinationConfig
sourceId: number
- pipelineConfig: {
- publicationName: string
- batch?: BatchConfig
- maxTableSyncWorkers?: number
- maxCopyConnectionsPerTable?: number
- invalidatedSlotBehavior?: 'error' | 'recreate'
- }
+ pipelineConfig: PipelineConfig
}
type UpdateDestinationPipelineBody =
@@ -40,13 +35,7 @@ async function updateDestinationPipeline(
projectRef,
destinationName: destinationName,
destinationConfig,
- pipelineConfig: {
- publicationName,
- batch,
- maxTableSyncWorkers,
- maxCopyConnectionsPerTable,
- invalidatedSlotBehavior,
- },
+ pipelineConfig,
sourceId,
}: UpdateDestinationPipelineParams,
signal?: AbortSignal
@@ -126,13 +115,7 @@ async function updateDestinationPipeline(
)
}
- const pipeline_config = {
- publication_name: publicationName,
- max_table_sync_workers: maxTableSyncWorkers,
- max_copy_connections_per_table: maxCopyConnectionsPerTable,
- invalidated_slot_behavior: invalidatedSlotBehavior,
- batch: batch ? { max_fill_ms: batch.maxFillMs } : undefined,
- }
+ const pipeline_config = buildPipelineApiConfig(pipelineConfig)
const { data, error } = await post(
'/platform/replication/{ref}/destinations-pipelines/{destination_id}/{pipeline_id}',
diff --git a/apps/studio/data/replication/validate-destination-mutation.ts b/apps/studio/data/replication/validate-destination-mutation.ts
index 327d108cea1..3e263ac79b3 100644
--- a/apps/studio/data/replication/validate-destination-mutation.ts
+++ b/apps/studio/data/replication/validate-destination-mutation.ts
@@ -1,7 +1,11 @@
import { useMutation } from '@tanstack/react-query'
import type { components } from 'api-types'
-import { buildDucklakeApiConfig, DestinationConfig } from './create-destination-pipeline-mutation'
+import {
+ buildDucklakeApiConfig,
+ DestinationConfig,
+ TableSyncCopyConfig,
+} from './create-destination-pipeline-mutation'
import { handleError, post } from '@/data/fetchers'
import type { ResponseError, UseCustomMutationOptions } from '@/types'
@@ -14,6 +18,7 @@ type ValidateDestinationParams = {
maxTableSyncWorkers?: number
maxCopyConnectionsPerTable?: number
invalidatedSlotBehavior?: 'error' | 'recreate'
+ tableSyncCopy?: TableSyncCopyConfig
}
type ValidateDestinationResponse = components['schemas']['ValidateDestinationResponse']
@@ -29,6 +34,7 @@ async function validateDestination(
maxTableSyncWorkers,
maxCopyConnectionsPerTable,
invalidatedSlotBehavior,
+ tableSyncCopy,
}: ValidateDestinationParams,
signal?: AbortSignal
): Promise<ValidateDestinationResponse> {
@@ -120,6 +126,7 @@ async function validateDestination(
max_table_sync_workers: maxTableSyncWorkers,
max_copy_connections_per_table: maxCopyConnectionsPerTable,
invalidated_slot_behavior: invalidatedSlotBehavior,
+ table_sync_copy: tableSyncCopy,
batch: batchConfig,
}
diff --git a/apps/studio/data/replication/validate-pipeline-mutation.ts b/apps/studio/data/replication/validate-pipeline-mutation.ts
index 1c90216766b..81fc461046a 100644
--- a/apps/studio/data/replication/validate-pipeline-mutation.ts
+++ b/apps/studio/data/replication/validate-pipeline-mutation.ts
@@ -1,6 +1,7 @@
import { useMutation } from '@tanstack/react-query'
import { components } from 'api-types'
+import type { TableSyncCopyConfig } from './create-destination-pipeline-mutation'
import { handleError, post } from '@/data/fetchers'
import type { ResponseError, UseCustomMutationOptions } from '@/types'
@@ -12,6 +13,7 @@ type ValidatePipelineParams = {
maxTableSyncWorkers?: number
maxCopyConnectionsPerTable?: number
invalidatedSlotBehavior?: 'error' | 'recreate'
+ tableSyncCopy: TableSyncCopyConfig
}
type ValidatePipelineResponse = components['schemas']['ValidatePipelineResponse']
@@ -24,6 +26,7 @@ async function validatePipeline(
maxTableSyncWorkers,
maxCopyConnectionsPerTable,
invalidatedSlotBehavior,
+ tableSyncCopy,
}: ValidatePipelineParams,
signal?: AbortSignal
): Promise<ValidatePipelineResponse> {
@@ -37,6 +40,7 @@ async function validatePipeline(
max_table_sync_workers: maxTableSyncWorkers,
max_copy_connections_per_table: maxCopyConnectionsPerTable,
invalidated_slot_behavior: invalidatedSlotBehavior,
+ table_sync_copy: tableSyncCopy,
batch: batchConfig,
}
diff --git a/packages/api-types/types/platform.d.ts b/packages/api-types/types/platform.d.ts
index f3fa019f046..8ddbcaaecf4 100644
--- a/packages/api-types/types/platform.d.ts
+++ b/packages/api-types/types/platform.d.ts
@@ -5862,7 +5862,7 @@ export interface components {
* @description Number of concurrent BigQuery Storage Write API connections.
* @example 8
*/
- connection_pool_size?: number
+ connection_pool_size?: number | null
/**
* @description BigQuery dataset id
* @example analytics
@@ -5872,7 +5872,7 @@ export interface components {
* @description Maximum data staleness in minutes
* @example 5
*/
- max_staleness_mins?: number
+ max_staleness_mins?: number | null
/**
* @description BigQuery project id
* @example my-gcp-project
@@ -5896,7 +5896,7 @@ export interface components {
*/
engine?: 'merge_tree' | 'replacing_merge_tree'
/** @description ClickHouse password. Omit for passwordless access. */
- password?: string
+ password?: string | null
/**
* @description ClickHouse HTTPS endpoint URL
* @example https://clickhouse.example.com:8443
@@ -5921,7 +5921,7 @@ export interface components {
* @description Namespace
* @example my-namespace
*/
- namespace?: string
+ namespace?: string | null
/**
* @description Project ref
* @example abcdefghijklmnopqrst
@@ -5973,33 +5973,33 @@ export interface components {
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/**
* @description S3-compatible storage access key ID
* @example my-access-key
*/
- s3_access_key_id: string
+ s3_access_key_id?: string | null
/**
* @description S3-compatible storage endpoint
* @example 127.0.0.1:5000/s3
*/
- s3_endpoint: string
+ s3_endpoint?: string | null
/**
* @description S3-compatible storage region
* @example us-east-1
*/
- s3_region: string
+ s3_region?: string | null
/**
* @description S3-compatible storage secret access key
* @example my-secret-key
*/
- s3_secret_access_key: string
+ s3_secret_access_key?: string | null
/**
* @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
* @example path
- * @enum {string}
+ * @enum {string|null}
*/
- s3_url_style?: 'path' | 'vhost'
+ s3_url_style?: 'path' | 'vhost' | null
/**
* @description Whether to use SSL for S3-compatible storage
* @default true
@@ -6019,12 +6019,12 @@ export interface components {
* @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/** @enum {string} */
type: 'postgres_url'
}
@@ -6033,12 +6033,12 @@ export interface components {
* @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/**
* @description Supabase project ref used as the DuckLake PostgreSQL catalog
* @example abcjuqabhgwjjutfvtpa
@@ -6057,7 +6057,7 @@ export interface components {
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
storage:
| {
/**
@@ -6088,9 +6088,9 @@ export interface components {
/**
* @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
* @example path
- * @enum {string}
+ * @enum {string|null}
*/
- s3_url_style?: 'path' | 'vhost'
+ s3_url_style?: 'path' | 'vhost' | null
/**
* @description Whether to use SSL for S3-compatible storage
* @default true
@@ -6141,12 +6141,12 @@ export interface components {
/** @description Snowflake RSA private key PEM contents */
private_key: string
/** @description Optional passphrase for encrypted private keys */
- private_key_passphrase?: string
+ private_key_passphrase?: string | null
/**
* @description Optional Snowflake role
* @example ETL_ROLE
*/
- role?: string
+ role?: string | null
/**
* @description Snowflake target schema
* @example PUBLIC
@@ -6174,7 +6174,7 @@ export interface components {
* @description Number of concurrent BigQuery Storage Write API connections.
* @example 8
*/
- connection_pool_size?: number
+ connection_pool_size?: number | null
/**
* @description BigQuery dataset id
* @example analytics
@@ -6184,7 +6184,7 @@ export interface components {
* @description Maximum data staleness in minutes
* @example 5
*/
- max_staleness_mins?: number
+ max_staleness_mins?: number | null
/**
* @description BigQuery project id
* @example my-gcp-project
@@ -6208,7 +6208,7 @@ export interface components {
*/
engine?: 'merge_tree' | 'replacing_merge_tree'
/** @description ClickHouse password. Omit for passwordless access. */
- password?: string
+ password?: string | null
/**
* @description ClickHouse HTTPS endpoint URL
* @example https://clickhouse.example.com:8443
@@ -6233,7 +6233,7 @@ export interface components {
* @description Namespace
* @example my-namespace
*/
- namespace?: string
+ namespace?: string | null
/**
* @description Project ref
* @example abcdefghijklmnopqrst
@@ -6285,33 +6285,33 @@ export interface components {
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/**
* @description S3-compatible storage access key ID
* @example my-access-key
*/
- s3_access_key_id: string
+ s3_access_key_id?: string | null
/**
* @description S3-compatible storage endpoint
* @example 127.0.0.1:5000/s3
*/
- s3_endpoint: string
+ s3_endpoint?: string | null
/**
* @description S3-compatible storage region
* @example us-east-1
*/
- s3_region: string
+ s3_region?: string | null
/**
* @description S3-compatible storage secret access key
* @example my-secret-key
*/
- s3_secret_access_key: string
+ s3_secret_access_key?: string | null
/**
* @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
* @example path
- * @enum {string}
+ * @enum {string|null}
*/
- s3_url_style?: 'path' | 'vhost'
+ s3_url_style?: 'path' | 'vhost' | null
/**
* @description Whether to use SSL for S3-compatible storage
* @default true
@@ -6331,12 +6331,12 @@ export interface components {
* @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/** @enum {string} */
type: 'postgres_url'
}
@@ -6345,12 +6345,12 @@ export interface components {
* @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/**
* @description Supabase project ref used as the DuckLake PostgreSQL catalog
* @example abcjuqabhgwjjutfvtpa
@@ -6369,7 +6369,7 @@ export interface components {
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
storage:
| {
/**
@@ -6400,9 +6400,9 @@ export interface components {
/**
* @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
* @example path
- * @enum {string}
+ * @enum {string|null}
*/
- s3_url_style?: 'path' | 'vhost'
+ s3_url_style?: 'path' | 'vhost' | null
/**
* @description Whether to use SSL for S3-compatible storage
* @default true
@@ -6453,12 +6453,12 @@ export interface components {
/** @description Snowflake RSA private key PEM contents */
private_key: string
/** @description Optional passphrase for encrypted private keys */
- private_key_passphrase?: string
+ private_key_passphrase?: string | null
/**
* @description Optional Snowflake role
* @example ETL_ROLE
*/
- role?: string
+ role?: string | null
/**
* @description Snowflake target schema
* @example PUBLIC
@@ -6478,29 +6478,111 @@ export interface components {
destination_name: string
/** @description Pipeline configuration */
pipeline_config: {
- /** @description Batch configuration */
+ /** @description Batch configuration. Omit or use `null` for server defaults. */
batch?: {
+ /**
+ * @description Maximum preferred byte size for one source batch
+ * @example 8388608
+ */
+ max_bytes?: number
/**
* @description Maximum fill time in milliseconds
* @example 200
*/
max_fill_ms?: number
- }
+ /**
+ * @description Ratio of process memory reserved for incoming stream batch bytes, in (0, 1]
+ * @example 0.2
+ */
+ memory_budget_ratio?: number
+ } | null
/**
- * @description Behavior when the replication slot is invalidated
+ * @description Behavior when the replication slot is invalidated. `error` prevents pipeline startup and requires manual intervention. `recreate` automatically recreates the slot and restarts replication from scratch.
* @example error
- * @enum {string}
+ * @enum {string|null}
*/
- invalidated_slot_behavior?: 'error' | 'recreate'
+ invalidated_slot_behavior?: 'error' | 'recreate' | null
+ /**
+ * @description Replicator log level
+ * @example info
+ * @enum {string|null}
+ */
+ log_level?: 'debug' | 'error' | 'info' | 'trace' | 'warn' | null
/** @description Maximum number of copy connections per table */
- max_copy_connections_per_table?: number
+ max_copy_connections_per_table?: number | null
/** @description Maximum number of table sync workers */
- max_table_sync_workers?: number
+ max_table_sync_workers?: number | null
+ /** @description Memory-based backpressure configuration. Omission enables server defaults; `null` disables it. */
+ memory_backpressure?: {
+ /**
+ * @description Memory usage ratio above which backpressure is activated
+ * @example 0.85
+ */
+ activate_threshold: number
+ /**
+ * @description Memory usage ratio below which backpressure is released
+ * @example 0.75
+ */
+ resume_threshold: number
+ } | null
+ /**
+ * @description Milliseconds between one memory usage refresh and another
+ * @example 100
+ */
+ memory_refresh_interval_ms?: number | null
/**
* @description Publication name
* @example pub_orders
*/
publication_name: string
+ /**
+ * @description Milliseconds between one replication lag refresh and another
+ * @example 10000
+ */
+ replication_lag_refresh_interval_ms?: number | null
+ /**
+ * @description Milliseconds between one retry and another for timed worker retries
+ * @example 10000
+ */
+ table_error_retry_delay_ms?: number | null
+ /**
+ * @description Maximum number of automatic timed retry attempts
+ * @example 5
+ */
+ table_error_retry_max_attempts?: number | null
+ /** @description Selection rules for tables participating in initial table copy. Omit or use `null` to copy all tables. */
+ table_sync_copy?:
+ | {
+ /** @enum {string} */
+ type: 'include_all_tables'
+ }
+ | {
+ /** @enum {string} */
+ type: 'skip_all_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be performed
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'include_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be skipped
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'skip_tables'
+ }
+ | (never | null)
}
/**
* @description Source id
@@ -6511,29 +6593,111 @@ export interface components {
CreateReplicationPipelineBody: {
/** @description Pipeline configuration */
config: {
- /** @description Batch configuration */
+ /** @description Batch configuration. Omit or use `null` for server defaults. */
batch?: {
+ /**
+ * @description Maximum preferred byte size for one source batch
+ * @example 8388608
+ */
+ max_bytes?: number
/**
* @description Maximum fill time in milliseconds
* @example 200
*/
max_fill_ms?: number
- }
+ /**
+ * @description Ratio of process memory reserved for incoming stream batch bytes, in (0, 1]
+ * @example 0.2
+ */
+ memory_budget_ratio?: number
+ } | null
/**
- * @description Behavior when the replication slot is invalidated
+ * @description Behavior when the replication slot is invalidated. `error` prevents pipeline startup and requires manual intervention. `recreate` automatically recreates the slot and restarts replication from scratch.
* @example error
- * @enum {string}
+ * @enum {string|null}
*/
- invalidated_slot_behavior?: 'error' | 'recreate'
+ invalidated_slot_behavior?: 'error' | 'recreate' | null
+ /**
+ * @description Replicator log level
+ * @example info
+ * @enum {string|null}
+ */
+ log_level?: 'debug' | 'error' | 'info' | 'trace' | 'warn' | null
/** @description Maximum number of copy connections per table */
- max_copy_connections_per_table?: number
+ max_copy_connections_per_table?: number | null
/** @description Maximum number of table sync workers */
- max_table_sync_workers?: number
+ max_table_sync_workers?: number | null
+ /** @description Memory-based backpressure configuration. Omission enables server defaults; `null` disables it. */
+ memory_backpressure?: {
+ /**
+ * @description Memory usage ratio above which backpressure is activated
+ * @example 0.85
+ */
+ activate_threshold: number
+ /**
+ * @description Memory usage ratio below which backpressure is released
+ * @example 0.75
+ */
+ resume_threshold: number
+ } | null
+ /**
+ * @description Milliseconds between one memory usage refresh and another
+ * @example 100
+ */
+ memory_refresh_interval_ms?: number | null
/**
* @description Publication name
* @example pub_orders
*/
publication_name: string
+ /**
+ * @description Milliseconds between one replication lag refresh and another
+ * @example 10000
+ */
+ replication_lag_refresh_interval_ms?: number | null
+ /**
+ * @description Milliseconds between one retry and another for timed worker retries
+ * @example 10000
+ */
+ table_error_retry_delay_ms?: number | null
+ /**
+ * @description Maximum number of automatic timed retry attempts
+ * @example 5
+ */
+ table_error_retry_max_attempts?: number | null
+ /** @description Selection rules for tables participating in initial table copy. Omit or use `null` to copy all tables. */
+ table_sync_copy?:
+ | {
+ /** @enum {string} */
+ type: 'include_all_tables'
+ }
+ | {
+ /** @enum {string} */
+ type: 'skip_all_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be performed
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'include_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be skipped
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'skip_tables'
+ }
+ | (never | null)
}
/**
* @description Destination id
@@ -9933,7 +10097,7 @@ export interface components {
* @description Number of concurrent BigQuery Storage Write API connections.
* @example 8
*/
- connection_pool_size?: number
+ connection_pool_size: number
/**
* @description BigQuery dataset id
* @example analytics
@@ -9949,8 +10113,6 @@ export interface components {
* @example my-gcp-project
*/
project_id: string
- /** @description BigQuery service account key */
- service_account_key: string
}
}
| {
@@ -9965,12 +10127,10 @@ export interface components {
* @example replacing_merge_tree
* @enum {string}
*/
- engine?: 'merge_tree' | 'replacing_merge_tree'
- /** @description ClickHouse password. Omit for passwordless access. */
- password?: string
+ engine: 'merge_tree' | 'replacing_merge_tree'
/**
- * @description ClickHouse HTTPS endpoint URL
- * @example https://clickhouse.example.com:8443
+ * @description Persisted ClickHouse HTTP(S) endpoint URL
+ * @example http://clickhouse.example.com:8123
*/
url: string
/**
@@ -9981,546 +10141,284 @@ export interface components {
}
}
| {
- iceberg: {
- supabase: {
+ iceberg:
+ | {
+ supabase: {
+ /**
+ * @description Namespace
+ * @example my-namespace
+ */
+ namespace?: string
+ /**
+ * @description Project ref
+ * @example abcdefghijklmnopqrst
+ */
+ project_ref: string
+ /**
+ * @description S3 region
+ * @example ap-southeast-1
+ */
+ s3_region: string
+ /**
+ * @description Warehouse name
+ * @example my-warehouse
+ */
+ warehouse_name: string
+ }
+ }
+ | {
+ rest: {
+ /**
+ * @description Iceberg REST catalog URI
+ * @example https://catalog.example.com
+ */
+ catalog_uri: string
+ /**
+ * @description Namespace
+ * @example my-namespace
+ */
+ namespace?: string
+ /**
+ * @description S3-compatible storage endpoint
+ * @example https://s3.example.com
+ */
+ s3_endpoint: string
+ /**
+ * @description Warehouse name
+ * @example my-warehouse
+ */
+ warehouse_name: string
+ }
+ }
+ }
+ | {
+ ducklake: {
+ /**
+ * @description Persisted DuckLake data path
+ * @example s3://<bucket-name>/
+ */
+ data_path: string
+ /**
+ * @description Persisted DuckLake metadata schema
+ * @example ducklake
+ */
+ metadata_schema?: string
+ /**
+ * @description Number of concurrent DuckDB connections.
+ * @example 4
+ */
+ pool_size: number
+ /**
+ * @description Persisted S3-compatible storage endpoint
+ * @example 127.0.0.1:5000/s3
+ */
+ s3_endpoint?: string
+ /**
+ * @description S3-compatible storage region
+ * @example us-east-1
+ */
+ s3_region?: string
+ /**
+ * @description Persisted S3 URL style
+ * @example path
+ */
+ s3_url_style?: string
+ /**
+ * @description Whether to use SSL for S3-compatible storage
+ * @example false
+ */
+ s3_use_ssl?: boolean
+ }
+ }
+ | {
+ snowflake: {
+ /**
+ * @description Snowflake account identifier
+ * @example MYORG-MYACCOUNT
+ */
+ account_id: string
+ /**
+ * @description Snowflake target database
+ * @example ANALYTICS
+ */
+ database: string
+ /**
+ * @description Optional Snowflake role
+ * @example ETL_ROLE
+ */
+ role?: string
+ /**
+ * @description Snowflake target schema
+ * @example PUBLIC
+ */
+ schema: string
+ /**
+ * @description Snowflake user configured for key-pair authentication
+ * @example ETL_USER
+ */
+ user: string
+ }
+ }
+ /**
+ * @description Destination id
+ * @example 2001
+ */
+ id: number
+ /**
+ * @description Destination name
+ * @example bq-analytics
+ */
+ name: string
+ /**
+ * @description Tenant id
+ * @example tenant_9f3a2c
+ */
+ tenant_id: string
+ }
+ ReplicationDestinationsResponse: {
+ /** @description List of destinations */
+ destinations: {
+ /** @description Destination configuration */
+ config:
+ | {
+ big_query: {
/**
- * @description Catalog token
- * @example A jwt secret
+ * @description Number of concurrent BigQuery Storage Write API connections.
+ * @example 8
*/
- catalog_token: string
+ connection_pool_size: number
/**
- * @description Namespace
- * @example my-namespace
+ * @description BigQuery dataset id
+ * @example analytics
*/
- namespace?: string
+ dataset_id: string
/**
- * @description Project ref
- * @example abcdefghijklmnopqrst
+ * @description Maximum data staleness in minutes
+ * @example 5
*/
- project_ref: string
+ max_staleness_mins?: number
/**
- * @description S3 access key ID
- * @example 53383b1d0cdb16a3afa63152656aa3cc
+ * @description BigQuery project id
+ * @example my-gcp-project
*/
- s3_access_key_id: string
+ project_id: string
+ }
+ }
+ | {
+ clickhouse: {
/**
- * @description S3 region
- * @example ap-southeast-1
+ * @description ClickHouse target database
+ * @example analytics
*/
- s3_region: string
+ database: string
/**
- * @description S3 secret access key
- * @example 25a0c5e69d847088a3e6ffb901adf4d19bbf74a400dec2ee49f46401039b3258
+ * @description Table engine used for replicated tables. Defaults to `replacing_merge_tree` server-side when omitted.
+ * @example replacing_merge_tree
+ * @enum {string}
*/
- s3_secret_access_key: string
+ engine: 'merge_tree' | 'replacing_merge_tree'
/**
- * @description Warehouse name
- * @example my-warehouse
+ * @description Persisted ClickHouse HTTP(S) endpoint URL
+ * @example http://clickhouse.example.com:8123
*/
- warehouse_name: string
+ url: string
+ /**
+ * @description ClickHouse user name
+ * @example default
+ */
+ user: string
}
}
- }
- | {
- ducklake:
- | {
- /**
- * @description DuckLake catalog URL
- * @example postgres://user:pass@localhost:5432/ducklake_catalog
- */
- catalog_url: string
- /**
- * @description DuckLake data path
- * @example s3://<bucket-name>/
- */
- data_path: string
- /**
- * @description Schema used for DuckLake metadata tables stored in PostgreSQL
- * @default ducklake
- * @example ducklake
- */
- metadata_schema?: string
- /**
- * @description Number of concurrent DuckDB connections.
- * @example 4
- */
- pool_size?: number
- /**
- * @description S3-compatible storage access key ID
- * @example my-access-key
- */
- s3_access_key_id: string
- /**
- * @description S3-compatible storage endpoint
- * @example 127.0.0.1:5000/s3
- */
- s3_endpoint: string
- /**
- * @description S3-compatible storage region
- * @example us-east-1
- */
- s3_region: string
- /**
- * @description S3-compatible storage secret access key
- * @example my-secret-key
- */
- s3_secret_access_key: string
- /**
- * @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
- * @example path
- * @enum {string}
- */
- s3_url_style?: 'path' | 'vhost'
- /**
- * @description Whether to use SSL for S3-compatible storage
- * @default true
- * @example false
- */
- s3_use_ssl?: boolean
- }
- | {
- catalog:
- | {
- /**
- * @description DuckLake catalog URL
- * @example postgres://user:pass@localhost:5432/ducklake_catalog
- */
- catalog_url: string
- /**
- * @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
- * @example ducklake
- */
- metadata_schema?: string
- /**
- * @description Number of concurrent DuckDB connections.
- * @example 4
- */
- pool_size?: number
- /** @enum {string} */
- type: 'postgres_url'
- }
- | {
- /**
- * @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
- * @example ducklake
- */
- metadata_schema?: string
- /**
- * @description Number of concurrent DuckDB connections.
- * @example 4
- */
- pool_size?: number
- /**
- * @description Supabase project ref used as the DuckLake PostgreSQL catalog
- * @example abcjuqabhgwjjutfvtpa
- */
- project_ref: string
- /** @enum {string} */
- type: 'supabase_project'
- }
- /**
- * @description Schema used for DuckLake metadata tables stored in PostgreSQL
- * @default ducklake
- * @example ducklake
- */
- metadata_schema?: string
- /**
- * @description Number of concurrent DuckDB connections.
- * @example 4
- */
- pool_size?: number
- storage:
- | {
- /**
- * @description DuckLake data path
- * @example s3://<bucket-name>/
- */
- data_path: string
- /**
- * @description S3-compatible storage access key ID
- * @example my-access-key
- */
- s3_access_key_id: string
- /**
- * @description S3-compatible storage endpoint
- * @example 127.0.0.1:5000/s3
- */
- s3_endpoint: string
- /**
- * @description S3-compatible storage region
- * @example us-east-1
- */
- s3_region: string
- /**
- * @description S3-compatible storage secret access key
- * @example my-secret-key
- */
- s3_secret_access_key: string
- /**
- * @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
- * @example path
- * @enum {string}
- */
- s3_url_style?: 'path' | 'vhost'
- /**
- * @description Whether to use SSL for S3-compatible storage
- * @default true
- * @example false
- */
- s3_use_ssl?: boolean
- /** @enum {string} */
- type: 's3'
- }
- | {
- /**
- * @description Storage bucket id used for DuckLake data files
- * @example ducklake
- */
- bucket: string
- /**
- * @description Optional description for the generated Supabase Storage S3 credential
- * @example DuckLake replication destination
- */
- credential_description?: string
- /**
- * @description Optional path prefix inside the bucket for DuckLake data files
- * @example replication
- */
- path?: string
- /**
- * @description Supabase project ref whose Object Storage S3 API stores DuckLake data files
- * @example abcjuqabhgwjjutfvtpa
- */
- project_ref: string
- /** @enum {string} */
- type: 'supabase_storage'
- }
- }
- }
- | {
- snowflake: {
- /**
- * @description Snowflake account identifier
- * @example MYORG-MYACCOUNT
- */
- account_id: string
- /**
- * @description Snowflake target database
- * @example ANALYTICS
- */
- database: string
- /** @description Snowflake RSA private key PEM contents */
- private_key: string
- /** @description Optional passphrase for encrypted private keys */
- private_key_passphrase?: string
- /**
- * @description Optional Snowflake role
- * @example ETL_ROLE
- */
- role?: string
- /**
- * @description Snowflake target schema
- * @example PUBLIC
- */
- schema: string
- /**
- * @description Snowflake user configured for key-pair authentication
- * @example ETL_USER
- */
- user: string
+ | {
+ iceberg:
+ | {
+ supabase: {
+ /**
+ * @description Namespace
+ * @example my-namespace
+ */
+ namespace?: string
+ /**
+ * @description Project ref
+ * @example abcdefghijklmnopqrst
+ */
+ project_ref: string
+ /**
+ * @description S3 region
+ * @example ap-southeast-1
+ */
+ s3_region: string
+ /**
+ * @description Warehouse name
+ * @example my-warehouse
+ */
+ warehouse_name: string
+ }
+ }
+ | {
+ rest: {
+ /**
+ * @description Iceberg REST catalog URI
+ * @example https://catalog.example.com
+ */
+ catalog_uri: string
+ /**
+ * @description Namespace
+ * @example my-namespace
+ */
+ namespace?: string
+ /**
+ * @description S3-compatible storage endpoint
+ * @example https://s3.example.com
+ */
+ s3_endpoint: string
+ /**
+ * @description Warehouse name
+ * @example my-warehouse
+ */
+ warehouse_name: string
+ }
+ }
}
- }
- /**
- * @description Destination id
- * @example 2001
- */
- id: number
- /**
- * @description Destination name
- * @example bq-analytics
- */
- name: string
- /**
- * @description Tenant id
- * @example tenant_9f3a2c
- */
- tenant_id: string
- }
- ReplicationDestinationsResponse: {
- /** @description List of destinations */
- destinations: {
- /** @description Destination configuration */
- config:
| {
- big_query: {
+ ducklake: {
/**
- * @description Number of concurrent BigQuery Storage Write API connections.
- * @example 8
+ * @description Persisted DuckLake data path
+ * @example s3://<bucket-name>/
*/
- connection_pool_size?: number
+ data_path: string
/**
- * @description BigQuery dataset id
- * @example analytics
+ * @description Persisted DuckLake metadata schema
+ * @example ducklake
*/
- dataset_id: string
+ metadata_schema?: string
/**
- * @description Maximum data staleness in minutes
- * @example 5
+ * @description Number of concurrent DuckDB connections.
+ * @example 4
*/
- max_staleness_mins?: number
+ pool_size: number
/**
- * @description BigQuery project id
- * @example my-gcp-project
+ * @description Persisted S3-compatible storage endpoint
+ * @example 127.0.0.1:5000/s3
*/
- project_id: string
- /** @description BigQuery service account key */
- service_account_key: string
- }
- }
- | {
- clickhouse: {
+ s3_endpoint?: string
/**
- * @description ClickHouse target database
- * @example analytics
+ * @description S3-compatible storage region
+ * @example us-east-1
*/
- database: string
+ s3_region?: string
/**
- * @description Table engine used for replicated tables. Defaults to `replacing_merge_tree` server-side when omitted.
- * @example replacing_merge_tree
- * @enum {string}
- */
- engine?: 'merge_tree' | 'replacing_merge_tree'
- /** @description ClickHouse password. Omit for passwordless access. */
- password?: string
- /**
- * @description ClickHouse HTTPS endpoint URL
- * @example https://clickhouse.example.com:8443
+ * @description Persisted S3 URL style
+ * @example path
*/
- url: string
+ s3_url_style?: string
/**
- * @description ClickHouse user name
- * @example default
+ * @description Whether to use SSL for S3-compatible storage
+ * @example false
*/
- user: string
- }
- }
- | {
- iceberg: {
- supabase: {
- /**
- * @description Catalog token
- * @example A jwt secret
- */
- catalog_token: string
- /**
- * @description Namespace
- * @example my-namespace
- */
- namespace?: string
- /**
- * @description Project ref
- * @example abcdefghijklmnopqrst
- */
- project_ref: string
- /**
- * @description S3 access key ID
- * @example 53383b1d0cdb16a3afa63152656aa3cc
- */
- s3_access_key_id: string
- /**
- * @description S3 region
- * @example ap-southeast-1
- */
- s3_region: string
- /**
- * @description S3 secret access key
- * @example 25a0c5e69d847088a3e6ffb901adf4d19bbf74a400dec2ee49f46401039b3258
- */
- s3_secret_access_key: string
- /**
- * @description Warehouse name
- * @example my-warehouse
- */
- warehouse_name: string
- }
+ s3_use_ssl?: boolean
}
}
- | {
- ducklake:
- | {
- /**
- * @description DuckLake catalog URL
- * @example postgres://user:pass@localhost:5432/ducklake_catalog
- */
- catalog_url: string
- /**
- * @description DuckLake data path
- * @example s3://<bucket-name>/
- */
- data_path: string
- /**
- * @description Schema used for DuckLake metadata tables stored in PostgreSQL
- * @default ducklake
- * @example ducklake
- */
- metadata_schema?: string
- /**
- * @description Number of concurrent DuckDB connections.
- * @example 4
- */
- pool_size?: number
- /**
- * @description S3-compatible storage access key ID
- * @example my-access-key
- */
- s3_access_key_id: string
- /**
- * @description S3-compatible storage endpoint
- * @example 127.0.0.1:5000/s3
- */
- s3_endpoint: string
- /**
- * @description S3-compatible storage region
- * @example us-east-1
- */
- s3_region: string
- /**
- * @description S3-compatible storage secret access key
- * @example my-secret-key
- */
- s3_secret_access_key: string
- /**
- * @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
- * @example path
- * @enum {string}
- */
- s3_url_style?: 'path' | 'vhost'
- /**
- * @description Whether to use SSL for S3-compatible storage
- * @default true
- * @example false
- */
- s3_use_ssl?: boolean
- }
- | {
- catalog:
- | {
- /**
- * @description DuckLake catalog URL
- * @example postgres://user:pass@localhost:5432/ducklake_catalog
- */
- catalog_url: string
- /**
- * @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
- * @example ducklake
- */
- metadata_schema?: string
- /**
- * @description Number of concurrent DuckDB connections.
- * @example 4
- */
- pool_size?: number
- /** @enum {string} */
- type: 'postgres_url'
- }
- | {
- /**
- * @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
- * @example ducklake
- */
- metadata_schema?: string
- /**
- * @description Number of concurrent DuckDB connections.
- * @example 4
- */
- pool_size?: number
- /**
- * @description Supabase project ref used as the DuckLake PostgreSQL catalog
- * @example abcjuqabhgwjjutfvtpa
- */
- project_ref: string
- /** @enum {string} */
- type: 'supabase_project'
- }
- /**
- * @description Schema used for DuckLake metadata tables stored in PostgreSQL
- * @default ducklake
- * @example ducklake
- */
- metadata_schema?: string
- /**
- * @description Number of concurrent DuckDB connections.
- * @example 4
- */
- pool_size?: number
- storage:
- | {
- /**
- * @description DuckLake data path
- * @example s3://<bucket-name>/
- */
- data_path: string
- /**
- * @description S3-compatible storage access key ID
- * @example my-access-key
- */
- s3_access_key_id: string
- /**
- * @description S3-compatible storage endpoint
- * @example 127.0.0.1:5000/s3
- */
- s3_endpoint: string
- /**
- * @description S3-compatible storage region
- * @example us-east-1
- */
- s3_region: string
- /**
- * @description S3-compatible storage secret access key
- * @example my-secret-key
- */
- s3_secret_access_key: string
- /**
- * @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
- * @example path
- * @enum {string}
- */
- s3_url_style?: 'path' | 'vhost'
- /**
- * @description Whether to use SSL for S3-compatible storage
- * @default true
- * @example false
- */
- s3_use_ssl?: boolean
- /** @enum {string} */
- type: 's3'
- }
- | {
- /**
- * @description Storage bucket id used for DuckLake data files
- * @example ducklake
- */
- bucket: string
- /**
- * @description Optional description for the generated Supabase Storage S3 credential
- * @example DuckLake replication destination
- */
- credential_description?: string
- /**
- * @description Optional path prefix inside the bucket for DuckLake data files
- * @example replication
- */
- path?: string
- /**
- * @description Supabase project ref whose Object Storage S3 API stores DuckLake data files
- * @example abcjuqabhgwjjutfvtpa
- */
- project_ref: string
- /** @enum {string} */
- type: 'supabase_storage'
- }
- }
- }
| {
snowflake: {
/**
@@ -10533,10 +10431,6 @@ export interface components {
* @example ANALYTICS
*/
database: string
- /** @description Snowflake RSA private key PEM contents */
- private_key: string
- /** @description Optional passphrase for encrypted private keys */
- private_key_passphrase?: string
/**
* @description Optional Snowflake role
* @example ETL_ROLE
@@ -10609,7 +10503,7 @@ export interface components {
* @example reserved
* @enum {string}
*/
- wal_status?: 'reserved' | 'extended' | 'unreserved' | 'lost' | 'unknown'
+ wal_status?: 'extended' | 'lost' | 'reserved' | 'unknown' | 'unreserved'
/**
* @description Write lag expressed in milliseconds.
* @example 1500
@@ -10623,6 +10517,21 @@ export interface components {
pipeline_id: number
/** @description Table statuses */
table_statuses: {
+ /**
+ * @description Table id (Postgres OID)
+ * @example 16408
+ */
+ id: number
+ /**
+ * @description Table name
+ * @example orders
+ */
+ name: string
+ /**
+ * @description Table schema
+ * @example public
+ */
+ schema: string
/** @description Table replication state */
state:
| {
@@ -10666,12 +10575,14 @@ export interface components {
solution?: string
}
/**
- * @description Table id (Postgres OID)
+ * @deprecated
+ * @description Deprecated compatibility alias for `id`.
* @example 16408
*/
table_id: number
/**
- * @description Table name
+ * @deprecated
+ * @description Deprecated unquoted `schema.name` compatibility representation.
* @example public.orders
*/
table_name: string
@@ -10712,7 +10623,7 @@ export interface components {
* @example reserved
* @enum {string}
*/
- wal_status?: 'reserved' | 'extended' | 'unreserved' | 'lost' | 'unknown'
+ wal_status?: 'extended' | 'lost' | 'reserved' | 'unknown' | 'unreserved'
/**
* @description Write lag expressed in milliseconds.
* @example 1500
@@ -10727,27 +10638,108 @@ export interface components {
config: {
/** @description Batch configuration */
batch?: {
+ /**
+ * @description Maximum preferred byte size for one source batch
+ * @example 8388608
+ */
+ max_bytes?: number
/**
* @description Maximum fill time in milliseconds
* @example 200
*/
max_fill_ms?: number
+ /**
+ * @description Persisted ratio of process memory reserved for incoming stream batch bytes
+ * @example 0.2
+ */
+ memory_budget_ratio?: number
}
/**
- * @description Behavior when the replication slot is invalidated
+ * @description Behavior when the replication slot is invalidated. `error` prevents pipeline startup and requires manual intervention. `recreate` automatically recreates the slot and restarts replication from scratch.
* @example error
* @enum {string}
*/
invalidated_slot_behavior?: 'error' | 'recreate'
+ /**
+ * @description Replicator log level
+ * @example info
+ * @enum {string}
+ */
+ log_level?: 'debug' | 'error' | 'info' | 'trace' | 'warn'
/** @description Maximum number of copy connections per table */
max_copy_connections_per_table?: number
/** @description Maximum number of table sync workers */
max_table_sync_workers?: number
+ /** @description Memory-based backpressure configuration */
+ memory_backpressure?: {
+ /**
+ * @description Persisted memory usage ratio that activates backpressure
+ * @example 0.85
+ */
+ activate_threshold: number
+ /**
+ * @description Persisted memory usage ratio that releases backpressure
+ * @example 0.75
+ */
+ resume_threshold: number
+ }
+ /**
+ * @description Milliseconds between one memory usage refresh and another
+ * @example 100
+ */
+ memory_refresh_interval_ms?: number
/**
* @description Publication name
* @example pub_orders
*/
publication_name: string
+ /**
+ * @description Milliseconds between one replication lag refresh and another
+ * @example 10000
+ */
+ replication_lag_refresh_interval_ms?: number
+ /**
+ * @description Milliseconds between one retry and another for timed worker retries
+ * @example 10000
+ */
+ table_error_retry_delay_ms?: number
+ /**
+ * @description Maximum number of automatic timed retry attempts
+ * @example 5
+ */
+ table_error_retry_max_attempts?: number
+ /** @description Selection rules for tables participating in initial table copy */
+ table_sync_copy?:
+ | {
+ /** @enum {string} */
+ type: 'include_all_tables'
+ }
+ | {
+ /** @enum {string} */
+ type: 'skip_all_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be performed
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'include_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be skipped
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'skip_tables'
+ }
}
/**
* @description Destination id
@@ -10792,27 +10784,108 @@ export interface components {
config: {
/** @description Batch configuration */
batch?: {
+ /**
+ * @description Maximum preferred byte size for one source batch
+ * @example 8388608
+ */
+ max_bytes?: number
/**
* @description Maximum fill time in milliseconds
* @example 200
*/
max_fill_ms?: number
+ /**
+ * @description Persisted ratio of process memory reserved for incoming stream batch bytes
+ * @example 0.2
+ */
+ memory_budget_ratio?: number
}
/**
- * @description Behavior when the replication slot is invalidated
+ * @description Behavior when the replication slot is invalidated. `error` prevents pipeline startup and requires manual intervention. `recreate` automatically recreates the slot and restarts replication from scratch.
* @example error
* @enum {string}
*/
invalidated_slot_behavior?: 'error' | 'recreate'
+ /**
+ * @description Replicator log level
+ * @example info
+ * @enum {string}
+ */
+ log_level?: 'debug' | 'error' | 'info' | 'trace' | 'warn'
/** @description Maximum number of copy connections per table */
max_copy_connections_per_table?: number
/** @description Maximum number of table sync workers */
max_table_sync_workers?: number
+ /** @description Memory-based backpressure configuration */
+ memory_backpressure?: {
+ /**
+ * @description Persisted memory usage ratio that activates backpressure
+ * @example 0.85
+ */
+ activate_threshold: number
+ /**
+ * @description Persisted memory usage ratio that releases backpressure
+ * @example 0.75
+ */
+ resume_threshold: number
+ }
+ /**
+ * @description Milliseconds between one memory usage refresh and another
+ * @example 100
+ */
+ memory_refresh_interval_ms?: number
/**
* @description Publication name
* @example pub_orders
*/
publication_name: string
+ /**
+ * @description Milliseconds between one replication lag refresh and another
+ * @example 10000
+ */
+ replication_lag_refresh_interval_ms?: number
+ /**
+ * @description Milliseconds between one retry and another for timed worker retries
+ * @example 10000
+ */
+ table_error_retry_delay_ms?: number
+ /**
+ * @description Maximum number of automatic timed retry attempts
+ * @example 5
+ */
+ table_error_retry_max_attempts?: number
+ /** @description Selection rules for tables participating in initial table copy */
+ table_sync_copy?:
+ | {
+ /** @enum {string} */
+ type: 'include_all_tables'
+ }
+ | {
+ /** @enum {string} */
+ type: 'skip_all_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be performed
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'include_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be skipped
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'skip_tables'
+ }
}
/**
* @description Destination id
@@ -10984,6 +11057,11 @@ export interface components {
name: string
/** @description Publication tables */
tables: {
+ /**
+ * @description Table id (Postgres OID)
+ * @example 16408
+ */
+ id: number
/**
* @description Table name
* @example orders
@@ -11043,6 +11121,11 @@ export interface components {
ReplicationTablesResponse: {
/** @description List of tables */
tables: {
+ /**
+ * @description Table id (Postgres OID)
+ * @example 16408
+ */
+ id: number
/**
* @description Table name
* @example orders
@@ -12423,24 +12506,24 @@ export interface components {
* @description Number of concurrent BigQuery Storage Write API connections.
* @example 8
*/
- connection_pool_size?: number
+ connection_pool_size?: number | null
/**
* @description BigQuery dataset id
* @example analytics
*/
- dataset_id: string
+ dataset_id?: string | null
/**
* @description Maximum data staleness in minutes
* @example 5
*/
- max_staleness_mins?: number
+ max_staleness_mins?: number | null
/**
* @description BigQuery project id
* @example my-gcp-project
*/
- project_id: string
+ project_id?: string | null
/** @description BigQuery service account key */
- service_account_key: string
+ service_account_key?: string | null
}
}
| {
@@ -12449,25 +12532,25 @@ export interface components {
* @description ClickHouse target database
* @example analytics
*/
- database: string
+ database?: string | null
/**
* @description Table engine used for replicated tables. Defaults to `replacing_merge_tree` server-side when omitted.
* @example replacing_merge_tree
- * @enum {string}
+ * @enum {string|null}
*/
- engine?: 'merge_tree' | 'replacing_merge_tree'
+ engine?: 'merge_tree' | 'replacing_merge_tree' | null
/** @description ClickHouse password. Omit for passwordless access. */
- password?: string
+ password?: string | null
/**
* @description ClickHouse HTTPS endpoint URL
* @example https://clickhouse.example.com:8443
*/
- url: string
+ url?: string | null
/**
* @description ClickHouse user name
* @example default
*/
- user: string
+ user?: string | null
}
}
| {
@@ -12477,37 +12560,37 @@ export interface components {
* @description Catalog token
* @example A jwt secret
*/
- catalog_token: string
+ catalog_token?: string | null
/**
* @description Namespace
* @example my-namespace
*/
- namespace?: string
+ namespace?: string | null
/**
* @description Project ref
* @example abcdefghijklmnopqrst
*/
- project_ref: string
+ project_ref?: string | null
/**
* @description S3 access key ID
* @example 53383b1d0cdb16a3afa63152656aa3cc
*/
- s3_access_key_id: string
+ s3_access_key_id?: string | null
/**
* @description S3 region
* @example ap-southeast-1
*/
- s3_region: string
+ s3_region?: string | null
/**
* @description S3 secret access key
* @example 25a0c5e69d847088a3e6ffb901adf4d19bbf74a400dec2ee49f46401039b3258
*/
- s3_secret_access_key: string
+ s3_secret_access_key?: string | null
/**
* @description Warehouse name
* @example my-warehouse
*/
- warehouse_name: string
+ warehouse_name?: string | null
}
}
}
@@ -12518,55 +12601,53 @@ export interface components {
* @description DuckLake catalog URL
* @example postgres://user:pass@localhost:5432/ducklake_catalog
*/
- catalog_url: string
+ catalog_url?: string | null
/**
* @description DuckLake data path
* @example s3://<bucket-name>/
*/
- data_path: string
+ data_path?: string | null
/**
* @description Schema used for DuckLake metadata tables stored in PostgreSQL
- * @default ducklake
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/**
* @description S3-compatible storage access key ID
* @example my-access-key
*/
- s3_access_key_id: string
+ s3_access_key_id?: string | null
/**
* @description S3-compatible storage endpoint
* @example 127.0.0.1:5000/s3
*/
- s3_endpoint: string
+ s3_endpoint?: string | null
/**
* @description S3-compatible storage region
* @example us-east-1
*/
- s3_region: string
+ s3_region?: string | null
/**
* @description S3-compatible storage secret access key
* @example my-secret-key
*/
- s3_secret_access_key: string
+ s3_secret_access_key?: string | null
/**
* @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
* @example path
- * @enum {string}
+ * @enum {string|null}
*/
- s3_url_style?: 'path' | 'vhost'
+ s3_url_style?: 'path' | 'vhost' | null
/**
* @description Whether to use SSL for S3-compatible storage
- * @default true
* @example false
*/
- s3_use_ssl?: boolean
+ s3_use_ssl?: boolean | null
}
| {
catalog:
@@ -12580,12 +12661,12 @@ export interface components {
* @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/** @enum {string} */
type: 'postgres_url'
}
@@ -12594,12 +12675,12 @@ export interface components {
* @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/**
* @description Supabase project ref used as the DuckLake PostgreSQL catalog
* @example abcjuqabhgwjjutfvtpa
@@ -12610,15 +12691,14 @@ export interface components {
}
/**
* @description Schema used for DuckLake metadata tables stored in PostgreSQL
- * @default ducklake
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
storage:
| {
/**
@@ -12649,15 +12729,14 @@ export interface components {
/**
* @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
* @example path
- * @enum {string}
+ * @enum {string|null}
*/
- s3_url_style?: 'path' | 'vhost'
+ s3_url_style?: 'path' | 'vhost' | null
/**
* @description Whether to use SSL for S3-compatible storage
- * @default true
* @example false
*/
- s3_use_ssl?: boolean
+ s3_use_ssl?: boolean | null
/** @enum {string} */
type: 's3'
}
@@ -12693,31 +12772,31 @@ export interface components {
* @description Snowflake account identifier
* @example MYORG-MYACCOUNT
*/
- account_id: string
+ account_id?: string | null
/**
* @description Snowflake target database
* @example ANALYTICS
*/
- database: string
+ database?: string | null
/** @description Snowflake RSA private key PEM contents */
- private_key: string
+ private_key?: string | null
/** @description Optional passphrase for encrypted private keys */
- private_key_passphrase?: string
+ private_key_passphrase?: string | null
/**
* @description Optional Snowflake role
* @example ETL_ROLE
*/
- role?: string
+ role?: string | null
/**
* @description Snowflake target schema
* @example PUBLIC
*/
- schema: string
+ schema?: string | null
/**
* @description Snowflake user configured for key-pair authentication
* @example ETL_USER
*/
- user: string
+ user?: string | null
}
}
/**
@@ -12735,24 +12814,24 @@ export interface components {
* @description Number of concurrent BigQuery Storage Write API connections.
* @example 8
*/
- connection_pool_size?: number
+ connection_pool_size?: number | null
/**
* @description BigQuery dataset id
* @example analytics
*/
- dataset_id: string
+ dataset_id?: string | null
/**
* @description Maximum data staleness in minutes
* @example 5
*/
- max_staleness_mins?: number
+ max_staleness_mins?: number | null
/**
* @description BigQuery project id
* @example my-gcp-project
*/
- project_id: string
+ project_id?: string | null
/** @description BigQuery service account key */
- service_account_key: string
+ service_account_key?: string | null
}
}
| {
@@ -12761,25 +12840,25 @@ export interface components {
* @description ClickHouse target database
* @example analytics
*/
- database: string
+ database?: string | null
/**
* @description Table engine used for replicated tables. Defaults to `replacing_merge_tree` server-side when omitted.
* @example replacing_merge_tree
- * @enum {string}
+ * @enum {string|null}
*/
- engine?: 'merge_tree' | 'replacing_merge_tree'
+ engine?: 'merge_tree' | 'replacing_merge_tree' | null
/** @description ClickHouse password. Omit for passwordless access. */
- password?: string
+ password?: string | null
/**
* @description ClickHouse HTTPS endpoint URL
* @example https://clickhouse.example.com:8443
*/
- url: string
+ url?: string | null
/**
* @description ClickHouse user name
* @example default
*/
- user: string
+ user?: string | null
}
}
| {
@@ -12789,37 +12868,37 @@ export interface components {
* @description Catalog token
* @example A jwt secret
*/
- catalog_token: string
+ catalog_token?: string | null
/**
* @description Namespace
* @example my-namespace
*/
- namespace?: string
+ namespace?: string | null
/**
* @description Project ref
* @example abcdefghijklmnopqrst
*/
- project_ref: string
+ project_ref?: string | null
/**
* @description S3 access key ID
* @example 53383b1d0cdb16a3afa63152656aa3cc
*/
- s3_access_key_id: string
+ s3_access_key_id?: string | null
/**
* @description S3 region
* @example ap-southeast-1
*/
- s3_region: string
+ s3_region?: string | null
/**
* @description S3 secret access key
* @example 25a0c5e69d847088a3e6ffb901adf4d19bbf74a400dec2ee49f46401039b3258
*/
- s3_secret_access_key: string
+ s3_secret_access_key?: string | null
/**
* @description Warehouse name
* @example my-warehouse
*/
- warehouse_name: string
+ warehouse_name?: string | null
}
}
}
@@ -12830,55 +12909,53 @@ export interface components {
* @description DuckLake catalog URL
* @example postgres://user:pass@localhost:5432/ducklake_catalog
*/
- catalog_url: string
+ catalog_url?: string | null
/**
* @description DuckLake data path
* @example s3://<bucket-name>/
*/
- data_path: string
+ data_path?: string | null
/**
* @description Schema used for DuckLake metadata tables stored in PostgreSQL
- * @default ducklake
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/**
* @description S3-compatible storage access key ID
* @example my-access-key
*/
- s3_access_key_id: string
+ s3_access_key_id?: string | null
/**
* @description S3-compatible storage endpoint
* @example 127.0.0.1:5000/s3
*/
- s3_endpoint: string
+ s3_endpoint?: string | null
/**
* @description S3-compatible storage region
* @example us-east-1
*/
- s3_region: string
+ s3_region?: string | null
/**
* @description S3-compatible storage secret access key
* @example my-secret-key
*/
- s3_secret_access_key: string
+ s3_secret_access_key?: string | null
/**
* @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
* @example path
- * @enum {string}
+ * @enum {string|null}
*/
- s3_url_style?: 'path' | 'vhost'
+ s3_url_style?: 'path' | 'vhost' | null
/**
* @description Whether to use SSL for S3-compatible storage
- * @default true
* @example false
*/
- s3_use_ssl?: boolean
+ s3_use_ssl?: boolean | null
}
| {
catalog:
@@ -12892,12 +12969,12 @@ export interface components {
* @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/** @enum {string} */
type: 'postgres_url'
}
@@ -12906,12 +12983,12 @@ export interface components {
* @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/**
* @description Supabase project ref used as the DuckLake PostgreSQL catalog
* @example abcjuqabhgwjjutfvtpa
@@ -12922,15 +12999,14 @@ export interface components {
}
/**
* @description Schema used for DuckLake metadata tables stored in PostgreSQL
- * @default ducklake
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
storage:
| {
/**
@@ -12961,15 +13037,14 @@ export interface components {
/**
* @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
* @example path
- * @enum {string}
+ * @enum {string|null}
*/
- s3_url_style?: 'path' | 'vhost'
+ s3_url_style?: 'path' | 'vhost' | null
/**
* @description Whether to use SSL for S3-compatible storage
- * @default true
* @example false
*/
- s3_use_ssl?: boolean
+ s3_use_ssl?: boolean | null
/** @enum {string} */
type: 's3'
}
@@ -13005,31 +13080,31 @@ export interface components {
* @description Snowflake account identifier
* @example MYORG-MYACCOUNT
*/
- account_id: string
+ account_id?: string | null
/**
* @description Snowflake target database
* @example ANALYTICS
*/
- database: string
+ database?: string | null
/** @description Snowflake RSA private key PEM contents */
- private_key: string
+ private_key?: string | null
/** @description Optional passphrase for encrypted private keys */
- private_key_passphrase?: string
+ private_key_passphrase?: string | null
/**
* @description Optional Snowflake role
* @example ETL_ROLE
*/
- role?: string
+ role?: string | null
/**
* @description Snowflake target schema
* @example PUBLIC
*/
- schema: string
+ schema?: string | null
/**
* @description Snowflake user configured for key-pair authentication
* @example ETL_USER
*/
- user: string
+ user?: string | null
}
}
/**
@@ -13037,31 +13112,113 @@ export interface components {
* @example bq-analytics
*/
destination_name: string
- /** @description Pipeline configuration */
+ /** @description Pipeline configuration patch */
pipeline_config: {
- /** @description Batch configuration */
+ /** @description Complete batch configuration replacement. Omission preserves it; `null` resets server defaults. */
batch?: {
+ /**
+ * @description Maximum preferred byte size for one source batch
+ * @example 8388608
+ */
+ max_bytes?: number
/**
* @description Maximum fill time in milliseconds
* @example 200
*/
max_fill_ms?: number
- }
+ /**
+ * @description Ratio of process memory reserved for incoming stream batch bytes, in (0, 1]
+ * @example 0.2
+ */
+ memory_budget_ratio?: number
+ } | null
/**
- * @description Behavior when the replication slot is invalidated
+ * @description Behavior when the replication slot is invalidated. `error` prevents pipeline startup and requires manual intervention. `recreate` automatically recreates the slot and restarts replication from scratch.
* @example error
- * @enum {string}
+ * @enum {string|null}
*/
- invalidated_slot_behavior?: 'error' | 'recreate'
+ invalidated_slot_behavior?: 'error' | 'recreate' | null
+ /**
+ * @description Replicator log level
+ * @example info
+ * @enum {string|null}
+ */
+ log_level?: 'debug' | 'error' | 'info' | 'trace' | 'warn' | null
/** @description Maximum number of copy connections per table */
- max_copy_connections_per_table?: number
+ max_copy_connections_per_table?: number | null
/** @description Maximum number of table sync workers */
- max_table_sync_workers?: number
+ max_table_sync_workers?: number | null
+ /** @description Complete memory backpressure replacement. Omission preserves it; `null` resets server defaults. */
+ memory_backpressure?: {
+ /**
+ * @description Memory usage ratio above which backpressure is activated
+ * @example 0.85
+ */
+ activate_threshold: number
+ /**
+ * @description Memory usage ratio below which backpressure is released
+ * @example 0.75
+ */
+ resume_threshold: number
+ } | null
/**
- * @description Publication name
+ * @description Milliseconds between one memory usage refresh and another
+ * @example 100
+ */
+ memory_refresh_interval_ms?: number | null
+ /**
+ * @description Publication name. Omission preserves the stored value; ETL rejects clearing it with `null`.
* @example pub_orders
*/
- publication_name: string
+ publication_name?: string | null
+ /**
+ * @description Milliseconds between one replication lag refresh and another
+ * @example 10000
+ */
+ replication_lag_refresh_interval_ms?: number | null
+ /**
+ * @description Milliseconds between one retry and another for timed worker retries
+ * @example 10000
+ */
+ table_error_retry_delay_ms?: number | null
+ /**
+ * @description Maximum number of automatic timed retry attempts
+ * @example 5
+ */
+ table_error_retry_max_attempts?: number | null
+ /** @description Initial-copy selection. Omission preserves it; `null` resets to copying all tables. */
+ table_sync_copy?:
+ | {
+ /** @enum {string} */
+ type: 'include_all_tables'
+ }
+ | {
+ /** @enum {string} */
+ type: 'skip_all_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be performed
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'include_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be skipped
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'skip_tables'
+ }
+ | (never | null)
}
/**
* @description Source id
@@ -13070,31 +13227,113 @@ export interface components {
source_id: number
}
UpdateReplicationPipelineBody: {
- /** @description Pipeline configuration */
+ /** @description Pipeline configuration patch */
config: {
- /** @description Batch configuration */
+ /** @description Complete batch configuration replacement. Omission preserves it; `null` resets server defaults. */
batch?: {
+ /**
+ * @description Maximum preferred byte size for one source batch
+ * @example 8388608
+ */
+ max_bytes?: number
/**
* @description Maximum fill time in milliseconds
* @example 200
*/
max_fill_ms?: number
- }
+ /**
+ * @description Ratio of process memory reserved for incoming stream batch bytes, in (0, 1]
+ * @example 0.2
+ */
+ memory_budget_ratio?: number
+ } | null
/**
- * @description Behavior when the replication slot is invalidated
+ * @description Behavior when the replication slot is invalidated. `error` prevents pipeline startup and requires manual intervention. `recreate` automatically recreates the slot and restarts replication from scratch.
* @example error
- * @enum {string}
+ * @enum {string|null}
*/
- invalidated_slot_behavior?: 'error' | 'recreate'
+ invalidated_slot_behavior?: 'error' | 'recreate' | null
+ /**
+ * @description Replicator log level
+ * @example info
+ * @enum {string|null}
+ */
+ log_level?: 'debug' | 'error' | 'info' | 'trace' | 'warn' | null
/** @description Maximum number of copy connections per table */
- max_copy_connections_per_table?: number
+ max_copy_connections_per_table?: number | null
/** @description Maximum number of table sync workers */
- max_table_sync_workers?: number
+ max_table_sync_workers?: number | null
+ /** @description Complete memory backpressure replacement. Omission preserves it; `null` resets server defaults. */
+ memory_backpressure?: {
+ /**
+ * @description Memory usage ratio above which backpressure is activated
+ * @example 0.85
+ */
+ activate_threshold: number
+ /**
+ * @description Memory usage ratio below which backpressure is released
+ * @example 0.75
+ */
+ resume_threshold: number
+ } | null
/**
- * @description Publication name
+ * @description Milliseconds between one memory usage refresh and another
+ * @example 100
+ */
+ memory_refresh_interval_ms?: number | null
+ /**
+ * @description Publication name. Omission preserves the stored value; ETL rejects clearing it with `null`.
* @example pub_orders
*/
- publication_name: string
+ publication_name?: string | null
+ /**
+ * @description Milliseconds between one replication lag refresh and another
+ * @example 10000
+ */
+ replication_lag_refresh_interval_ms?: number | null
+ /**
+ * @description Milliseconds between one retry and another for timed worker retries
+ * @example 10000
+ */
+ table_error_retry_delay_ms?: number | null
+ /**
+ * @description Maximum number of automatic timed retry attempts
+ * @example 5
+ */
+ table_error_retry_max_attempts?: number | null
+ /** @description Initial-copy selection. Omission preserves it; `null` resets to copying all tables. */
+ table_sync_copy?:
+ | {
+ /** @enum {string} */
+ type: 'include_all_tables'
+ }
+ | {
+ /** @enum {string} */
+ type: 'skip_all_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be performed
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'include_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be skipped
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'skip_tables'
+ }
+ | (never | null)
}
/**
* @description Destination id
@@ -13466,7 +13705,7 @@ export interface components {
* @description Number of concurrent BigQuery Storage Write API connections.
* @example 8
*/
- connection_pool_size?: number
+ connection_pool_size?: number | null
/**
* @description BigQuery dataset id
* @example analytics
@@ -13476,7 +13715,7 @@ export interface components {
* @description Maximum data staleness in minutes
* @example 5
*/
- max_staleness_mins?: number
+ max_staleness_mins?: number | null
/**
* @description BigQuery project id
* @example my-gcp-project
@@ -13500,7 +13739,7 @@ export interface components {
*/
engine?: 'merge_tree' | 'replacing_merge_tree'
/** @description ClickHouse password. Omit for passwordless access. */
- password?: string
+ password?: string | null
/**
* @description ClickHouse HTTPS endpoint URL
* @example https://clickhouse.example.com:8443
@@ -13525,7 +13764,7 @@ export interface components {
* @description Namespace
* @example my-namespace
*/
- namespace?: string
+ namespace?: string | null
/**
* @description Project ref
* @example abcdefghijklmnopqrst
@@ -13577,33 +13816,33 @@ export interface components {
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/**
* @description S3-compatible storage access key ID
* @example my-access-key
*/
- s3_access_key_id: string
+ s3_access_key_id?: string | null
/**
* @description S3-compatible storage endpoint
* @example 127.0.0.1:5000/s3
*/
- s3_endpoint: string
+ s3_endpoint?: string | null
/**
* @description S3-compatible storage region
* @example us-east-1
*/
- s3_region: string
+ s3_region?: string | null
/**
* @description S3-compatible storage secret access key
* @example my-secret-key
*/
- s3_secret_access_key: string
+ s3_secret_access_key?: string | null
/**
* @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
* @example path
- * @enum {string}
+ * @enum {string|null}
*/
- s3_url_style?: 'path' | 'vhost'
+ s3_url_style?: 'path' | 'vhost' | null
/**
* @description Whether to use SSL for S3-compatible storage
* @default true
@@ -13623,12 +13862,12 @@ export interface components {
* @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/** @enum {string} */
type: 'postgres_url'
}
@@ -13637,12 +13876,12 @@ export interface components {
* @description Schema used for DuckLake metadata tables stored in this PostgreSQL catalog. Overrides `metadata_schema` when provided.
* @example ducklake
*/
- metadata_schema?: string
+ metadata_schema?: string | null
/**
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
/**
* @description Supabase project ref used as the DuckLake PostgreSQL catalog
* @example abcjuqabhgwjjutfvtpa
@@ -13661,7 +13900,7 @@ export interface components {
* @description Number of concurrent DuckDB connections.
* @example 4
*/
- pool_size?: number
+ pool_size?: number | null
storage:
| {
/**
@@ -13692,9 +13931,9 @@ export interface components {
/**
* @description S3 URL style: `path` (MinIO/Supabase) or `vhost` (AWS)
* @example path
- * @enum {string}
+ * @enum {string|null}
*/
- s3_url_style?: 'path' | 'vhost'
+ s3_url_style?: 'path' | 'vhost' | null
/**
* @description Whether to use SSL for S3-compatible storage
* @default true
@@ -13745,12 +13984,12 @@ export interface components {
/** @description Snowflake RSA private key PEM contents */
private_key: string
/** @description Optional passphrase for encrypted private keys */
- private_key_passphrase?: string
+ private_key_passphrase?: string | null
/**
* @description Optional Snowflake role
* @example ETL_ROLE
*/
- role?: string
+ role?: string | null
/**
* @description Snowflake target schema
* @example PUBLIC
@@ -13765,29 +14004,111 @@ export interface components {
}
/** @description Pipeline configuration */
pipeline_config?: {
- /** @description Batch configuration */
+ /** @description Batch configuration. Omit or use `null` for server defaults. */
batch?: {
+ /**
+ * @description Maximum preferred byte size for one source batch
+ * @example 8388608
+ */
+ max_bytes?: number
/**
* @description Maximum fill time in milliseconds
* @example 200
*/
max_fill_ms?: number
- }
+ /**
+ * @description Ratio of process memory reserved for incoming stream batch bytes, in (0, 1]
+ * @example 0.2
+ */
+ memory_budget_ratio?: number
+ } | null
/**
- * @description Behavior when the replication slot is invalidated
+ * @description Behavior when the replication slot is invalidated. `error` prevents pipeline startup and requires manual intervention. `recreate` automatically recreates the slot and restarts replication from scratch.
* @example error
- * @enum {string}
+ * @enum {string|null}
*/
- invalidated_slot_behavior?: 'error' | 'recreate'
+ invalidated_slot_behavior?: 'error' | 'recreate' | null
+ /**
+ * @description Replicator log level
+ * @example info
+ * @enum {string|null}
+ */
+ log_level?: 'debug' | 'error' | 'info' | 'trace' | 'warn' | null
/** @description Maximum number of copy connections per table */
- max_copy_connections_per_table?: number
+ max_copy_connections_per_table?: number | null
/** @description Maximum number of table sync workers */
- max_table_sync_workers?: number
+ max_table_sync_workers?: number | null
+ /** @description Memory-based backpressure configuration. Omission enables server defaults; `null` disables it. */
+ memory_backpressure?: {
+ /**
+ * @description Memory usage ratio above which backpressure is activated
+ * @example 0.85
+ */
+ activate_threshold: number
+ /**
+ * @description Memory usage ratio below which backpressure is released
+ * @example 0.75
+ */
+ resume_threshold: number
+ } | null
+ /**
+ * @description Milliseconds between one memory usage refresh and another
+ * @example 100
+ */
+ memory_refresh_interval_ms?: number | null
/**
* @description Publication name
* @example pub_orders
*/
publication_name: string
+ /**
+ * @description Milliseconds between one replication lag refresh and another
+ * @example 10000
+ */
+ replication_lag_refresh_interval_ms?: number | null
+ /**
+ * @description Milliseconds between one retry and another for timed worker retries
+ * @example 10000
+ */
+ table_error_retry_delay_ms?: number | null
+ /**
+ * @description Maximum number of automatic timed retry attempts
+ * @example 5
+ */
+ table_error_retry_max_attempts?: number | null
+ /** @description Selection rules for tables participating in initial table copy. Omit or use `null` to copy all tables. */
+ table_sync_copy?:
+ | {
+ /** @enum {string} */
+ type: 'include_all_tables'
+ }
+ | {
+ /** @enum {string} */
+ type: 'skip_all_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be performed
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'include_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be skipped
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'skip_tables'
+ }
+ | (never | null)
}
/**
* @description Source id
@@ -13798,29 +14119,111 @@ export interface components {
ValidateReplicationPipelineBody: {
/** @description Pipeline configuration */
config: {
- /** @description Batch configuration */
+ /** @description Batch configuration. Omit or use `null` for server defaults. */
batch?: {
+ /**
+ * @description Maximum preferred byte size for one source batch
+ * @example 8388608
+ */
+ max_bytes?: number
/**
* @description Maximum fill time in milliseconds
* @example 200
*/
max_fill_ms?: number
- }
+ /**
+ * @description Ratio of process memory reserved for incoming stream batch bytes, in (0, 1]
+ * @example 0.2
+ */
+ memory_budget_ratio?: number
+ } | null
/**
- * @description Behavior when the replication slot is invalidated
+ * @description Behavior when the replication slot is invalidated. `error` prevents pipeline startup and requires manual intervention. `recreate` automatically recreates the slot and restarts replication from scratch.
* @example error
- * @enum {string}
+ * @enum {string|null}
*/
- invalidated_slot_behavior?: 'error' | 'recreate'
+ invalidated_slot_behavior?: 'error' | 'recreate' | null
+ /**
+ * @description Replicator log level
+ * @example info
+ * @enum {string|null}
+ */
+ log_level?: 'debug' | 'error' | 'info' | 'trace' | 'warn' | null
/** @description Maximum number of copy connections per table */
- max_copy_connections_per_table?: number
+ max_copy_connections_per_table?: number | null
/** @description Maximum number of table sync workers */
- max_table_sync_workers?: number
+ max_table_sync_workers?: number | null
+ /** @description Memory-based backpressure configuration. Omission enables server defaults; `null` disables it. */
+ memory_backpressure?: {
+ /**
+ * @description Memory usage ratio above which backpressure is activated
+ * @example 0.85
+ */
+ activate_threshold: number
+ /**
+ * @description Memory usage ratio below which backpressure is released
+ * @example 0.75
+ */
+ resume_threshold: number
+ } | null
+ /**
+ * @description Milliseconds between one memory usage refresh and another
+ * @example 100
+ */
+ memory_refresh_interval_ms?: number | null
/**
* @description Publication name
* @example pub_orders
*/
publication_name: string
+ /**
+ * @description Milliseconds between one replication lag refresh and another
+ * @example 10000
+ */
+ replication_lag_refresh_interval_ms?: number | null
+ /**
+ * @description Milliseconds between one retry and another for timed worker retries
+ * @example 10000
+ */
+ table_error_retry_delay_ms?: number | null
+ /**
+ * @description Maximum number of automatic timed retry attempts
+ * @example 5
+ */
+ table_error_retry_max_attempts?: number | null
+ /** @description Selection rules for tables participating in initial table copy. Omit or use `null` to copy all tables. */
+ table_sync_copy?:
+ | {
+ /** @enum {string} */
+ type: 'include_all_tables'
+ }
+ | {
+ /** @enum {string} */
+ type: 'skip_all_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be performed
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'include_tables'
+ }
+ | {
+ /**
+ * @description Table ids for which copy should be skipped
+ * @example [
+ * 16408
+ * ]
+ */
+ table_ids: number[]
+ /** @enum {string} */
+ type: 'skip_tables'
+ }
+ | (never | null)
}
/**
* @description Source id
diff --git a/packages/ui-patterns/src/multi-select/multi-select.test.tsx b/packages/ui-patterns/src/multi-select/multi-select.test.tsx
index 005a90e983f..65312eb342c 100644
--- a/packages/ui-patterns/src/multi-select/multi-select.test.tsx
+++ b/packages/ui-patterns/src/multi-select/multi-select.test.tsx
@@ -48,6 +48,16 @@ function MultiSelectDemo() {
}
describe('multi-select', () => {
+ it('renders selected values with a custom label', () => {
+ render(
+ <MultiSelector values={['101']} onValuesChange={() => undefined}>
+ <MultiSelectorTrigger renderValue={(value) => `public.table_${value}`} />
+ </MultiSelector>
+ )
+
+ expect(screen.getByRole('combobox')).toHaveTextContent('public.table_101')
+ })
+
it('opens the dropdown when the MultiSelectorTrigger is clicked', () => {
render(<MultiSelectDemo />)
diff --git a/packages/ui-patterns/src/multi-select/multi-select.tsx b/packages/ui-patterns/src/multi-select/multi-select.tsx
index 683f7758991..a168f84cf53 100644
--- a/packages/ui-patterns/src/multi-select/multi-select.tsx
+++ b/packages/ui-patterns/src/multi-select/multi-select.tsx
@@ -209,6 +209,7 @@ export interface MultiSelectorTriggerProps extends React.HTMLAttributes<HTMLButt
deletableBadge?: boolean
showIcon?: boolean
mode?: MultiSelectorMode
+ renderValue?: (value: string) => React.ReactNode
}
const MultiSelectorTrigger = React.forwardRef<HTMLButtonElement, MultiSelectorTriggerProps>(
@@ -221,6 +222,7 @@ const MultiSelectorTrigger = React.forwardRef<HTMLButtonElement, MultiSelectorTr
badgeLimit = 9999,
showIcon = true,
mode = 'combobox',
+ renderValue,
children,
...props
},
@@ -312,7 +314,7 @@ const MultiSelectorTrigger = React.forwardRef<HTMLButtonElement, MultiSelectorTr
>
{visibleBadges.map((value) => (
<Badge key={value} className={badgeClasses}>
- {value}
+ {renderValue?.(value) ?? value}
{deletableBadge && (
<div
onMouseEnter={() => setIsDeleteHovered(true)}