The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
feat(studio): configure BigQuery table layout (#49535)
Riccardo Busettidc16371a4736
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsxindex c3c4ae5b85e..472e5b427ef 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx@@ -19,6 +19,7 @@ import {import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'import { DestinationType } from '../DestinationPanel.types'+import { TableOptions } from './BigQuery/TableOptions'import {DEFAULT_CONNECTION_POOL_SIZE,DEFAULT_MAX_COPY_CONNECTIONS_PER_TABLE,@@ -237,6 +238,17 @@ export const AdvancedSettings = ({</FormItemLayout>)}/>++ <div className="flex flex-col gap-y-3">+ <div className="flex flex-col gap-y-1">+ <span className="text-sm text-foreground">Table layout</span>+ <p className="text-sm text-foreground-lighter">+ Partitioning and clustering for each BigQuery table. Applied when a+ destination table is first created or reset.+ </p>+ </div>+ <TableOptions control={form.control} />+ </div></>)}</AccordionContent>diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.schema.test.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.schema.test.tsnew file mode 100644index 00000000000..a5690c79acd--- /dev/null+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.schema.test.ts@@ -0,0 +1,100 @@+import { describe, expect, it } from 'vitest'++import { BigQueryPartitionBySchema, BigQueryTableOptionSchema } from './BigQuery.schema'++describe('BigQuery replication schemas', () => {+ it('accepts integer ranges at the JavaScript safe-integer boundary', () => {+ expect(+ BigQueryPartitionBySchema.safeParse({+ kind: 'integer_range',+ column: 'account_id',+ start: Number.MIN_SAFE_INTEGER,+ end: Number.MAX_SAFE_INTEGER,+ interval: 1,+ }).success+ ).toBe(true)+ })++ it('rejects integer ranges that cannot be represented exactly by JavaScript', () => {+ expect(+ BigQueryPartitionBySchema.safeParse({+ kind: 'integer_range',+ column: 'account_id',+ start: 0,+ end: Number.MAX_SAFE_INTEGER + 1,+ interval: 1,+ }).success+ ).toBe(false)+ })++ it('rejects table ids outside the PostgreSQL OID range', () => {+ expect(BigQueryTableOptionSchema.safeParse({ tableId: 4_294_967_296 }).success).toBe(false)+ })++ it('allows a selected table with no partitioning or clustering', () => {+ expect(BigQueryTableOptionSchema.safeParse({ tableId: 1 }).success).toBe(true)+ })++ it('rejects a column-based partition with no column instead of dropping it on save', () => {+ for (const partitionBy of [+ { kind: 'time_column', column: '', granularity: 'day' },+ { kind: 'integer_range', column: '', start: 0, end: 10, interval: 1 },+ ]) {+ const result = BigQueryTableOptionSchema.safeParse({ tableId: 1, partitionBy })++ expect(result.success).toBe(false)+ expect(result.error?.issues.map(({ message }) => message)).toContain(+ 'Select a partition column'+ )+ }+ })++ it('requires the integer range end to be greater than the start', () => {+ const result = BigQueryTableOptionSchema.safeParse({+ tableId: 1,+ partitionBy: { kind: 'integer_range', column: 'shard', start: 10, end: 10, interval: 1 },+ })++ expect(result.error?.issues).toContainEqual(+ expect.objectContaining({+ path: ['partitionBy', 'end'],+ message: 'End must be greater than start.',+ })+ )+ })++ it('requires a positive integer range interval', () => {+ const result = BigQueryTableOptionSchema.safeParse({+ tableId: 1,+ partitionBy: { kind: 'integer_range', column: 'shard', start: 0, end: 100, interval: 0 },+ })++ expect(result.error?.issues).toContainEqual(+ expect.objectContaining({+ path: ['partitionBy', 'interval'],+ message: 'Interval must be greater than 0.',+ })+ )+ })++ it('allows at most four clustering columns', () => {+ expect(+ BigQueryTableOptionSchema.safeParse({+ tableId: 1,+ clusterBy: ['one', 'two', 'three', 'four'],+ }).success+ ).toBe(true)++ const result = BigQueryTableOptionSchema.safeParse({+ tableId: 1,+ clusterBy: ['one', 'two', 'three', 'four', 'five'],+ })++ expect(result.error?.issues).toContainEqual(+ expect.objectContaining({+ path: ['clusterBy'],+ message: 'Select up to 4 clustering columns',+ })+ )+ })+})diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.schema.tsindex e7dcc7a3570..6756095fa15 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.schema.ts+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.schema.ts@@ -1,9 +1,83 @@import * as z from 'zod'+import {+ BIGQUERY_MAX_CLUSTERING_COLUMNS,+ BIGQUERY_TIME_PARTITION_GRANULARITIES,+} from '@/data/replication/types'import { optionalNumberInputSchema, requiredNumberInputSchema } from '@/lib/forms/zod-number-input'const CONNECTION_POOL_SIZE_MIN_ERROR = 'Connection pool size must be greater than 0.'+const BigQueryTimePartitionGranularitySchema = z.enum(BIGQUERY_TIME_PARTITION_GRANULARITIES)+const integerRangeValue = (label: string) =>+ z+ .union([+ z.literal(''),+ z.number().int(`${label} must be a whole number`).safe(`${label} must be a safe integer`),+ ])+ .refine((value): boolean => value !== '', `${label} is required`)++// A column-based partition is dropped from the save payload when its column is missing, so+// require one here rather than letting the form accept a layout it will silently discard.+const partitionColumn = z.string().min(1, 'Select a partition column')++export const BigQueryPartitionBySchema = z.discriminatedUnion('kind', [+ z.object({+ kind: z.literal('time_column'),+ column: partitionColumn,+ granularity: BigQueryTimePartitionGranularitySchema.optional(),+ }),+ z.object({+ kind: z.literal('integer_range'),+ column: partitionColumn,+ start: integerRangeValue('Start'),+ end: integerRangeValue('End'),+ interval: integerRangeValue('Interval'),+ }),+ z.object({+ kind: z.literal('ingestion_time'),+ granularity: BigQueryTimePartitionGranularitySchema.optional(),+ }),+])++export const BigQueryTableOptionSchema = z+ .object({+ tableId: z.number().int().nonnegative().max(4_294_967_295),+ partitionBy: BigQueryPartitionBySchema.optional(),+ clusterBy: z+ .array(z.string())+ .max(+ BIGQUERY_MAX_CLUSTERING_COLUMNS,+ `Select up to ${BIGQUERY_MAX_CLUSTERING_COLUMNS} clustering columns`+ )+ .optional(),+ })+ .superRefine((option, ctx) => {+ if (option.partitionBy?.kind !== 'integer_range') return+ if (option.partitionBy.column.trim().length === 0) return++ const { start, end, interval } = option.partitionBy+ if (typeof start !== 'number' || typeof end !== 'number' || typeof interval !== 'number') {+ return+ }++ if (start >= end) {+ ctx.addIssue({+ code: 'custom',+ path: ['partitionBy', 'end'],+ message: 'End must be greater than start.',+ })+ }++ if (interval <= 0) {+ ctx.addIssue({+ code: 'custom',+ path: ['partitionBy', 'interval'],+ message: 'Interval must be greater than 0.',+ })+ }+ })+export const BigQueryFormSchema = z.object({projectId: z.string().optional(),datasetId: z.string().optional(),@@ -23,4 +97,5 @@ export const BigQueryFormSchema = z.object({.int('Maximum staleness must be a whole number of minutes.').min(0, 'Maximum staleness must be 0 or greater.')),+ tableOptions: z.array(BigQueryTableOptionSchema).optional(),})diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.utils.test.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.utils.test.tsnew file mode 100644index 00000000000..321f805152b--- /dev/null+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.utils.test.ts@@ -0,0 +1,109 @@+import { describe, expect, it } from 'vitest'++import {+ defaultPartitionByForKind,+ getBigQueryValidationIssues,+ parseIntegerInput,+ shortenPgType,+} from './BigQuery.utils'++describe('getBigQueryValidationIssues', () => {+ const validData = {+ projectId: 'my-project',+ datasetId: 'analytics',+ serviceAccountKey: '{}',+ }++ it('returns no issues when all required fields are set', () => {+ expect(getBigQueryValidationIssues(validData)).toEqual([])+ })++ it('flags blank required fields', () => {+ expect(+ getBigQueryValidationIssues({ projectId: '', datasetId: ' ', serviceAccountKey: '{}' })+ ).toEqual([+ { path: 'projectId', message: 'Project ID is required.' },+ { path: 'datasetId', message: 'Dataset ID is required.' },+ ])+ })++ it('skips the service account key when secrets are optional (edit mode)', () => {+ expect(+ getBigQueryValidationIssues(+ { ...validData, serviceAccountKey: '' },+ { secretsOptional: true }+ )+ ).toEqual([])+ })+})++describe('shortenPgType', () => {+ it('shortens known verbose type names to their standard alias', () => {+ expect(shortenPgType('timestamp with time zone')).toBe('timestamptz')+ expect(shortenPgType('timestamp without time zone')).toBe('timestamp')+ expect(shortenPgType('character varying')).toBe('varchar')+ expect(shortenPgType('double precision')).toBe('float8')+ })++ it('returns unrecognized types unchanged', () => {+ expect(shortenPgType('bigint')).toBe('bigint')+ expect(shortenPgType('jsonb')).toBe('jsonb')+ })+})++describe('defaultPartitionByForKind', () => {+ it('returns undefined for "none"', () => {+ expect(defaultPartitionByForKind('none')).toBeUndefined()+ })++ it('defaults time_column with an empty column and day granularity', () => {+ expect(defaultPartitionByForKind('time_column')).toEqual({+ kind: 'time_column',+ column: '',+ granularity: 'day',+ })+ })++ it('defaults integer_range with a 0-100 range and interval of 10', () => {+ expect(defaultPartitionByForKind('integer_range')).toEqual({+ kind: 'integer_range',+ column: '',+ start: 0,+ end: 100,+ interval: 10,+ })+ })++ it('defaults ingestion_time with day granularity', () => {+ expect(defaultPartitionByForKind('ingestion_time')).toEqual({+ kind: 'ingestion_time',+ granularity: 'day',+ })+ })+})++describe('parseIntegerInput', () => {+ it('parses a valid integer string', () => {+ expect(parseIntegerInput('42', 0)).toBe(42)+ })++ it('parses a negative number', () => {+ expect(parseIntegerInput('-5', 0)).toBe(-5)+ })++ it('uses the empty sentinel for an empty string', () => {+ expect(parseIntegerInput('', 7)).toBe('')+ })++ it('uses the empty sentinel for whitespace-only input', () => {+ expect(parseIntegerInput(' ', 7)).toBe('')+ })++ it('keeps the previous value for non-numeric input', () => {+ expect(parseIntegerInput('abc', 7)).toBe(7)+ })++ it('uses the empty sentinel while a negative sign is typed alone', () => {+ expect(parseIntegerInput('-', 7)).toBe('')+ })+})diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.utils.tsindex 423632f5c4f..51bb86c3f32 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.utils.ts+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/BigQuery.utils.ts@@ -1,9 +1,51 @@import { type DestinationPanelSchemaType } from '../DestinationForm.schema'+import { BigQueryPartitionBy } from '@/data/replication/types'type BigQueryFieldPath = 'projectId' | 'datasetId' | 'serviceAccountKey'-export type BigQueryValidationIssue = {- path: BigQueryFieldPath+export type BigQueryPartitionKind = BigQueryPartitionBy['kind'] | 'none'++// Postgres' verbose spelling for a handful of common types, shown here as their much shorter,+// equally standard aliases so a column's type doesn't crowd out its name in a narrow picker.+const PG_TYPE_ALIASES: Record<string, string> = {+ 'timestamp with time zone': 'timestamptz',+ 'timestamp without time zone': 'timestamp',+ 'time with time zone': 'timetz',+ 'time without time zone': 'time',+ 'character varying': 'varchar',+ 'double precision': 'float8',+}++export const shortenPgType = (type: string) => PG_TYPE_ALIASES[type] ?? type++export const defaultPartitionByForKind = (+ kind: BigQueryPartitionKind+): BigQueryPartitionBy | undefined => {+ switch (kind) {+ case 'none':+ return undefined+ case 'time_column':+ return { kind: 'time_column', column: '', granularity: 'day' }+ case 'integer_range':+ return { kind: 'integer_range', column: '', start: 0, end: 100, interval: 10 }+ case 'ingestion_time':+ return { kind: 'ingestion_time', granularity: 'day' }+ }+}++// Parses a partition start/end/interval input. Empty and a lone minus stay as the empty+// sentinel so the field remains controlled while the user is still typing. Other invalid+// drafts keep the previous committed value.+export const parseIntegerInput = (value: string, previous: number | ''): number | '' => {+ const trimmed = value.trim()+ if (trimmed === '' || trimmed === '-') return ''+ if (!/^-?\d+$/.test(trimmed)) return previous+ const parsed = Number(trimmed)+ return Number.isSafeInteger(parsed) ? parsed : previous+}++export type BigQueryValidationIssue<Path extends string = string> = {+ path: Pathmessage: string}@@ -29,9 +71,9 @@ const isValidJsonString = (value: string) => {export const getBigQueryValidationIssues = (data: Pick<DestinationPanelSchemaType, BigQueryFieldPath>,options: { secretsOptional?: boolean; validateJson?: boolean } = {}-): BigQueryValidationIssue[] => {+): BigQueryValidationIssue<BigQueryFieldPath>[] => {const { secretsOptional = false, validateJson = true } = options- const issues: BigQueryValidationIssue[] = BIGQUERY_REQUIRED_FIELDS.filter(+ const issues: BigQueryValidationIssue<BigQueryFieldPath>[] = BIGQUERY_REQUIRED_FIELDS.filter(({ path }) => !data[path]?.trim().length).map(({ path, message }) => ({ path, message }))diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionFields.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionFields.tsxnew file mode 100644index 00000000000..40860b2acbd--- /dev/null+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionFields.tsx@@ -0,0 +1,332 @@+import type { ChangeEvent } from 'react'+import { Input, Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from 'ui'+import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'+import { MultiSelector } from 'ui-patterns/multi-select'+import { SelectionListState } from 'ui-patterns/SelectionListState'++import { defaultPartitionByForKind, parseIntegerInput, shortenPgType } from './BigQuery.utils'+import type { BigQueryPartitionKind } from './BigQuery.utils'+import { isPartitionColumnTypeCompatible } from './TableOptions.utils'+import {+ BIGQUERY_MAX_CLUSTERING_COLUMNS,+ BIGQUERY_TIME_PARTITION_GRANULARITIES,+ type BigQueryPartitionBy,+ type BigQueryTimePartitionGranularity,+} from '@/data/replication/types'++const PARTITION_KIND_LABELS: Record<BigQueryPartitionKind, string> = {+ none: 'None',+ time_column: 'Time column',+ integer_range: 'Integer range',+ ingestion_time: 'Ingestion time',+}++const GRANULARITY_LABELS: Record<BigQueryTimePartitionGranularity, string> = {+ hour: 'Hour',+ day: 'Day',+ month: 'Month',+ year: 'Year',+}++const ColumnOption = ({+ name,+ type,+ isUnavailable = false,+}: {+ name: string+ type?: string+ isUnavailable?: boolean+}) => (+ <span+ className={+ isUnavailable+ ? 'flex items-center gap-x-2 min-w-0 text-destructive-600'+ : 'flex items-center gap-x-2 min-w-0'+ }+ >+ <span className="truncate min-w-0">{name}</span>+ {isUnavailable && <span className="sr-only"> (no longer available)</span>}+ {type && (+ <span+ className={+ isUnavailable+ ? 'font-mono text-xs shrink-0 text-destructive-600'+ : 'font-mono text-xs shrink-0 text-foreground-lighter'+ }+ >+ {shortenPgType(type)}+ </span>+ )}+ </span>+)++export interface ColumnSelectionState {+ availableColumnNames: string[]+ columnTypeByName: ReadonlyMap<string, string>+ isError: boolean+ isLoading: boolean+ onOpenChange: (isOpen: boolean) => void+ unavailableColumnSet: ReadonlySet<string>+}++interface PartitioningFieldsProps {+ columnSelection: ColumnSelectionState+ errors: {+ column?: string+ end?: string+ interval?: string+ start?: string+ }+ onChange: (partitionBy: BigQueryPartitionBy | undefined) => void+ onNeedsColumns: () => void+ partitionBy: BigQueryPartitionBy | undefined+}++export const PartitioningFields = ({+ columnSelection,+ errors,+ onChange,+ onNeedsColumns,+ partitionBy,+}: PartitioningFieldsProps) => {+ const partitionKind: BigQueryPartitionKind = partitionBy?.kind ?? 'none'++ const compatibleColumnNames =+ partitionBy && (partitionBy.kind === 'time_column' || partitionBy.kind === 'integer_range')+ ? columnSelection.availableColumnNames.filter((column) =>+ isPartitionColumnTypeCompatible(+ partitionBy.kind,+ columnSelection.columnTypeByName.get(column)+ )+ )+ : columnSelection.availableColumnNames+ const compatibleColumnEmptyLabel =+ partitionBy?.kind === 'time_column'+ ? 'No published date or timestamp columns'+ : 'No published integer columns'++ return (+ <>+ <FormItemLayout isReactForm={false} label="Partition by" layout="vertical">+ <Select+ value={partitionKind}+ onValueChange={(kind: BigQueryPartitionKind) => {+ if (kind === 'time_column' || kind === 'integer_range') onNeedsColumns()+ onChange(defaultPartitionByForKind(kind))+ }}+ >+ <SelectTrigger>+ <SelectValue />+ </SelectTrigger>+ <SelectContent side="bottom" collisionPadding={16}>+ {Object.entries(PARTITION_KIND_LABELS).map(([kind, label]) => (+ <SelectItem key={kind} value={kind}>+ {label}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </FormItemLayout>++ {partitionBy && 'column' in partitionBy && (+ <FormItemLayout+ isReactForm={false}+ label="Partition column"+ layout="vertical"+ error={errors.column}+ >+ <Select+ value={partitionBy.column || undefined}+ onOpenChange={columnSelection.onOpenChange}+ onValueChange={(column: string) => onChange({ ...partitionBy, column })}+ >+ <SelectTrigger+ className={+ columnSelection.unavailableColumnSet.has(partitionBy.column)+ ? 'border-destructive-600'+ : undefined+ }+ >+ {partitionBy.column ? (+ <ColumnOption+ name={partitionBy.column}+ type={columnSelection.columnTypeByName.get(partitionBy.column)}+ isUnavailable={columnSelection.unavailableColumnSet.has(partitionBy.column)}+ />+ ) : (+ <SelectValue placeholder="Select a column" />+ )}+ </SelectTrigger>+ <SelectContent side="bottom" collisionPadding={16}>+ <SelectionListState+ isLoading={columnSelection.isLoading}+ isError={columnSelection.isError}+ isEmpty={+ !columnSelection.isLoading &&+ !columnSelection.isError &&+ compatibleColumnNames.length === 0+ }+ emptyLabel={compatibleColumnEmptyLabel}+ errorLabel="Unable to load columns. Open the list again to retry."+ />+ {compatibleColumnNames.map((column) => (+ <SelectItem key={column} value={column}>+ <ColumnOption name={column} type={columnSelection.columnTypeByName.get(column)} />+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </FormItemLayout>+ )}++ {partitionBy && 'granularity' in partitionBy && (+ <FormItemLayout isReactForm={false} label="Granularity" layout="vertical">+ <Select+ value={partitionBy.granularity ?? 'day'}+ onValueChange={(granularity: BigQueryTimePartitionGranularity) =>+ onChange({ ...partitionBy, granularity })+ }+ >+ <SelectTrigger>+ <SelectValue />+ </SelectTrigger>+ <SelectContent side="bottom" collisionPadding={16}>+ {BIGQUERY_TIME_PARTITION_GRANULARITIES.map((granularity) => (+ <SelectItem key={granularity} value={granularity}>+ {GRANULARITY_LABELS[granularity]}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </FormItemLayout>+ )}++ {partitionBy?.kind === 'integer_range' && (+ <div className="grid grid-cols-[repeat(auto-fit,minmax(10rem,1fr))] gap-2">+ <FormItemLayout+ isReactForm={false}+ label="Start"+ layout="vertical"+ description="Inclusive"+ error={errors.start}+ >+ <Input+ type="text"+ inputMode="numeric"+ value={partitionBy.start}+ onChange={(event: ChangeEvent<HTMLInputElement>) =>+ onChange({+ ...partitionBy,+ start: parseIntegerInput(event.target.value, partitionBy.start),+ })+ }+ />+ </FormItemLayout>+ <FormItemLayout+ isReactForm={false}+ label="End"+ layout="vertical"+ description="Exclusive"+ error={errors.end}+ >+ <Input+ type="text"+ inputMode="numeric"+ value={partitionBy.end}+ onChange={(event: ChangeEvent<HTMLInputElement>) =>+ onChange({+ ...partitionBy,+ end: parseIntegerInput(event.target.value, partitionBy.end),+ })+ }+ />+ </FormItemLayout>+ <FormItemLayout+ isReactForm={false}+ label="Interval"+ layout="vertical"+ description="Range width"+ error={errors.interval}+ >+ <Input+ type="text"+ inputMode="numeric"+ value={partitionBy.interval}+ onChange={(event: ChangeEvent<HTMLInputElement>) =>+ onChange({+ ...partitionBy,+ interval: parseIntegerInput(event.target.value, partitionBy.interval),+ })+ }+ />+ </FormItemLayout>+ </div>+ )}+ </>+ )+}++interface ClusteringFieldsProps {+ clusterBy: string[]+ columnSelection: ColumnSelectionState+ error?: string+ onChange: (columns: string[]) => void+}++export const ClusteringFields = ({+ clusterBy,+ columnSelection,+ error,+ onChange,+}: ClusteringFieldsProps) => {+ const hasReachedColumnLimit = clusterBy.length >= BIGQUERY_MAX_CLUSTERING_COLUMNS++ return (+ <FormItemLayout+ isReactForm={false}+ label="Cluster by"+ layout="vertical"+ description={`Selection order sets the clustering order. Maximum ${BIGQUERY_MAX_CLUSTERING_COLUMNS} columns.`}+ error={error}+ >+ <MultiSelector+ values={clusterBy}+ onValuesChange={(columns) => {+ if (columns.length <= BIGQUERY_MAX_CLUSTERING_COLUMNS) onChange(columns)+ }}+ onOpenChange={columnSelection.onOpenChange}+ >+ <MultiSelector.Trigger+ aria-label="Select clustering columns"+ badgeLimit="wrap"+ label="Select columns..."+ renderValue={(column) => (+ <ColumnOption+ name={column}+ type={columnSelection.columnTypeByName.get(column)}+ isUnavailable={columnSelection.unavailableColumnSet.has(column)}+ />+ )}+ />+ <MultiSelector.Content>+ <MultiSelector.List+ emptyLabel="No published columns available"+ error={columnSelection.isError}+ errorLabel="Unable to load columns. Open the list again to retry."+ loading={columnSelection.isLoading}+ >+ {columnSelection.availableColumnNames.map((column) => (+ <MultiSelector.Item+ key={column}+ value={column}+ disabled={hasReachedColumnLimit && !clusterBy.includes(column)}+ >+ <ColumnOption name={column} type={columnSelection.columnTypeByName.get(column)} />+ </MultiSelector.Item>+ ))}+ </MultiSelector.List>+ </MultiSelector.Content>+ </MultiSelector>+ </FormItemLayout>+ )+}diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionRow.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionRow.tsxnew file mode 100644index 00000000000..f1f7d91caec--- /dev/null+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptionRow.tsx@@ -0,0 +1,259 @@+import { useParams } from 'common'+import { RotateCcw } from 'lucide-react'+import { useState } from 'react'+import { get, useController, useFormState, type Control, type FieldErrors } from 'react-hook-form'+import { Button } from 'ui'+import { Admonition } from 'ui-patterns/Admonition'++import type { DestinationPanelSchemaType } from '../DestinationForm.schema'+import {+ isMetadataListErrorVisible,+ isMetadataListLoading,+ useRefreshOnOpen,+} from '../useRefreshOnOpen'+import {+ ClusteringFields,+ PartitioningFields,+ type ColumnSelectionState,+} from './TableOptionFields'+import { useReplicationSourceId } from '@/data/replication/sources-query'+import { useReplicationTableColumnsQuery } from '@/data/replication/table-columns-query'++type TableOption = NonNullable<DestinationPanelSchemaType['tableOptions']>[number]+type PartitionBy = TableOption['partitionBy']++const getFieldErrorMessage = (errors: FieldErrors<DestinationPanelSchemaType>, path: string) => {+ const error = get(errors, path)+ return typeof error?.message === 'string' ? error.message : undefined+}++interface TableOptionErrors {+ partitionBy?: string+ column?: string+ start?: string+ end?: string+ interval?: string+ clusterBy?: string+}++interface TableOptionEditorProps {+ partitionBy: PartitionBy+ clusterBy: string[]+ onPartitionByChange: (partitionBy: PartitionBy) => void+ onClusterByChange: (clusterBy: string[]) => void+ errors?: TableOptionErrors+ isPublicationColumnsError?: boolean+ isPublicationColumnsPending?: boolean+ publishedColumnNames?: ReadonlySet<string> | null+ tableId: number+ onClear?: () => void+}++const TableOptionEditor = ({+ partitionBy,+ clusterBy,+ onPartitionByChange,+ onClusterByChange,+ errors = {},+ isPublicationColumnsError = false,+ isPublicationColumnsPending = false,+ publishedColumnNames,+ tableId,+ onClear,+}: TableOptionEditorProps) => {+ const { ref: projectRef } = useParams()+ const sourceId = useReplicationSourceId({ projectRef })++ const shouldInitiallyLoadColumns =+ partitionBy?.kind === 'time_column' ||+ partitionBy?.kind === 'integer_range' ||+ clusterBy.length > 0+ const [shouldLoadColumns, setShouldLoadColumns] = useState(shouldInitiallyLoadColumns)+ const {+ data: columns = [],+ isPending,+ isFetching,+ isError,+ isSuccess,+ refetch: refetchColumns,+ } = useReplicationTableColumnsQuery(+ { projectRef, sourceId, tableId },+ { enabled: shouldLoadColumns }+ )++ const isLoadingColumns =+ (shouldLoadColumns && isMetadataListLoading(isPending || isFetching, columns.length)) ||+ isPublicationColumnsPending+ const sourceColumnNames = columns.map((column) => column.name)+ const sourceColumnNameSet = new Set(sourceColumnNames)+ const canUseColumns = !isPublicationColumnsPending && !isPublicationColumnsError+ const availableColumnNames = canUseColumns+ ? sourceColumnNames.filter(+ (column) =>+ publishedColumnNames === undefined ||+ (publishedColumnNames !== null && publishedColumnNames.has(column))+ )+ : []+ const columnTypeByName = new Map(columns.map((column) => [column.name, column.type]))+ const areColumnsVerified = isSuccess && canUseColumns+ const partitionColumn = partitionBy && 'column' in partitionBy ? partitionBy.column : undefined+ const configuredColumns = [partitionColumn, ...clusterBy].filter(+ (column, columnIndex, allConfiguredColumns): column is string =>+ typeof column === 'string' &&+ column.trim().length > 0 &&+ allConfiguredColumns.indexOf(column) === columnIndex+ )+ const unavailableColumns = configuredColumns.filter(+ (column) =>+ areColumnsVerified &&+ (!sourceColumnNameSet.has(column) ||+ (publishedColumnNames != null && !publishedColumnNames.has(column)))+ )+ const unavailableColumnSet = new Set(unavailableColumns)+ const isColumnSelectionError =+ isPublicationColumnsError || isMetadataListErrorVisible(isError, columns.length)++ const { handleOpenChange: handleRefreshColumnsOnOpen } = useRefreshOnOpen({+ isEnabled: shouldLoadColumns,+ refetch: refetchColumns,+ })+ const handleColumnPickerOpenChange = (isOpen: boolean) => {+ if (isOpen && !shouldLoadColumns) {+ setShouldLoadColumns(true)+ return+ }+ handleRefreshColumnsOnOpen(isOpen)+ }++ const columnSelection: ColumnSelectionState = {+ availableColumnNames,+ columnTypeByName,+ isError: isColumnSelectionError,+ isLoading: isLoadingColumns,+ onOpenChange: handleColumnPickerOpenChange,+ unavailableColumnSet,+ }++ return (+ <div className="flex flex-col gap-y-4 border-t px-3 py-4">+ <PartitioningFields+ partitionBy={partitionBy}+ onChange={onPartitionByChange}+ onNeedsColumns={() => setShouldLoadColumns(true)}+ columnSelection={columnSelection}+ errors={{+ column: errors.column,+ start: errors.start,+ end: errors.end,+ interval: errors.interval,+ }}+ />++ {errors.partitionBy && <p className="text-sm text-destructive-600">{errors.partitionBy}</p>}++ <ClusteringFields+ clusterBy={clusterBy}+ onChange={onClusterByChange}+ columnSelection={columnSelection}+ error={errors.clusterBy}+ />++ {shouldLoadColumns && isColumnSelectionError && (+ <Admonition+ type="warning"+ title="Columns could not be verified"+ description="Refresh the page before changing or saving this table’s layout."+ />+ )}++ {unavailableColumns.length > 0 && (+ <Admonition+ type="destructive"+ title="Some selected columns are no longer available"+ description="Choose columns that are still in the source and publication."+ />+ )}++ {onClear !== undefined && (+ <Button+ type="button"+ variant="default"+ className="self-start"+ icon={<RotateCcw size={14} />}+ onClick={onClear}+ >+ Clear+ </Button>+ )}+ </div>+ )+}++type SharedRowProps = Pick<+ TableOptionEditorProps,+ 'isPublicationColumnsError' | 'isPublicationColumnsPending' | 'publishedColumnNames' | 'tableId'+>++interface TableOptionRowProps extends SharedRowProps {+ control: Control<DestinationPanelSchemaType>+ index: number+ onClear: () => void+}++export const TableOptionRow = ({+ control,+ index,+ onClear,+ ...sharedProps+}: TableOptionRowProps) => {+ const { errors } = useFormState({ control })++ const { field: partitionByField } = useController({+ control,+ name: `tableOptions.${index}.partitionBy`,+ })+ const { field: clusterByField } = useController({+ control,+ name: `tableOptions.${index}.clusterBy`,+ })++ const fieldPath = `tableOptions.${index}`++ return (+ <TableOptionEditor+ partitionBy={partitionByField.value}+ clusterBy={clusterByField.value ?? []}+ onPartitionByChange={partitionByField.onChange}+ onClusterByChange={clusterByField.onChange}+ errors={{+ partitionBy: getFieldErrorMessage(errors, `${fieldPath}.partitionBy`),+ column: getFieldErrorMessage(errors, `${fieldPath}.partitionBy.column`),+ start: getFieldErrorMessage(errors, `${fieldPath}.partitionBy.start`),+ end: getFieldErrorMessage(errors, `${fieldPath}.partitionBy.end`),+ interval: getFieldErrorMessage(errors, `${fieldPath}.partitionBy.interval`),+ clusterBy: getFieldErrorMessage(errors, `${fieldPath}.clusterBy`),+ }}+ onClear={onClear}+ {...sharedProps}+ />+ )+}++interface TableOptionDraftRowProps extends SharedRowProps {+ onCreate: (option: Pick<TableOption, 'partitionBy' | 'clusterBy'>) => void+}++/**+ * Rendered while a table has no entry in the tableOptions field array. Expanding a row must not+ * touch form state, otherwise merely looking at a table leaves the form dirty and triggers the+ * unsaved-changes prompt. The first edit creates the entry, after which TableOptionRow takes over.+ */+export const TableOptionDraftRow = ({ onCreate, ...sharedProps }: TableOptionDraftRowProps) => (+ <TableOptionEditor+ partitionBy={undefined}+ clusterBy={[]}+ onPartitionByChange={(partitionBy) => onCreate({ partitionBy, clusterBy: [] })}+ onClusterByChange={(clusterBy) => onCreate({ partitionBy: undefined, clusterBy })}+ {...sharedProps}+ />+)diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.test.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.test.tsxnew file mode 100644index 00000000000..711757428c3--- /dev/null+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.test.tsx@@ -0,0 +1,505 @@+import { zodResolver } from '@hookform/resolvers/zod'+import { fireEvent, screen } from '@testing-library/react'+import type { components } from 'api-types'+import { HttpResponse } from 'msw'+import { useForm } from 'react-hook-form'+import { Form } from 'ui'+import { describe, expect, it } from 'vitest'++import {+ DestinationPanelFormSchema,+ type DestinationPanelSchemaType,+} from '../DestinationForm.schema'+import { TableOptions } from './TableOptions'+import { customRender } from '@/tests/lib/custom-render'+import { addAPIMock } from '@/tests/lib/msw'++type PublicationDetailsResponse = components['schemas']['PublicationDetailsResponse_Output']+type ReadColumnsResponse = components['schemas']['ReadColumnsResponse_Output']+type ReadTablesResponse = components['schemas']['ReadTablesResponse_Output']+type SourcesResponse = components['schemas']['SourcesResponse_Output']++const publicationTable = {+ id: 101,+ schema: 'public',+ name: 'Orders',+ kind: 'table' as const,+ partition_parent_id: null,+}++const publication: PublicationDetailsResponse = {+ name: 'analytics',+ config: {+ type: 'tables',+ tables: [+ {+ id: publicationTable.id,+ schema: publicationTable.schema,+ name: publicationTable.name,+ columns: null,+ row_filter: null,+ },+ ],+ operations: ['insert'],+ publish_via_partition_root: false,+ },+ tables: [publicationTable],+}++const mockSources = () => {+ const response: SourcesResponse = {+ sources: [+ {+ id: 1,+ name: 'default',+ tenant_id: 'tenant',+ config: { host: 'db.internal', name: 'main-db', port: 5432, username: 'etl_user' },+ },+ ],+ }++ addAPIMock({+ method: 'get',+ path: '/platform/replication/:ref/sources',+ response: () => HttpResponse.json<SourcesResponse>(response),+ })+}++const mockPublication = (response = publication) => {+ addAPIMock({+ method: 'get',+ path: '/platform/replication/v2/:ref/sources/:source_id/publications/:publication_name',+ response: () => HttpResponse.json<PublicationDetailsResponse>(response),+ })+}++const mockColumns = (tableId: number, columns: ReadColumnsResponse['columns']) => {+ addAPIMock({+ method: 'get',+ path: '/platform/replication/v2/:ref/sources/:source_id/tables/:table_id/columns',+ response: ({ params }) => {+ if (Number(params.table_id) !== tableId) {+ return HttpResponse.json<ReadColumnsResponse>({ columns: [] })+ }+ return HttpResponse.json<ReadColumnsResponse>({ columns })+ },+ })+}++const mockTables = (tables: ReadTablesResponse['tables']) => {+ addAPIMock({+ method: 'get',+ path: '/platform/replication/v2/:ref/sources/:source_id/tables',+ response: () => HttpResponse.json<ReadTablesResponse>({ tables }),+ })+}++const TableOptionsHarness = ({+ tableOptions,+ publicationName = 'analytics',+}: {+ tableOptions: NonNullable<DestinationPanelSchemaType['tableOptions']>+ publicationName?: string+}) => {+ const form = useForm<DestinationPanelSchemaType>({+ mode: 'onChange',+ resolver: zodResolver(DestinationPanelFormSchema),+ defaultValues: {+ name: 'Warehouse',+ publicationName,+ tableSyncCopyMode: 'include_all_tables',+ tableSyncCopyTableIds: [],+ tableOptions,+ },+ })++ return (+ <Form {...form}>+ <TableOptions control={form.control} />+ <p>{form.formState.isDirty ? 'Form is dirty' : 'Form is pristine'}</p>+ <button type="button" tabIndex={0} onClick={() => void form.trigger()}>+ Validate form+ </button>+ </Form>+ )+}++describe('TableOptions source reconciliation', () => {+ it('explains why a publication is required before showing table settings', async () => {+ mockSources()++ customRender(<TableOptionsHarness publicationName="" tableOptions={[]} />)++ expect(await screen.findByText('Select a publication')).toBeInTheDocument()+ expect(+ screen.getByText(+ 'Choose the publication whose destination tables you want to partition or cluster.'+ )+ ).toBeInTheDocument()+ })++ it('discloses table settings without implying that the table is excluded', async () => {+ mockSources()+ mockPublication()++ customRender(<TableOptionsHarness tableOptions={[]} />)++ const tableTrigger = await screen.findByRole('button', {+ name: 'public.Orders: Not configured',+ })+ expect(screen.queryByText('Partition by')).not.toBeInTheDocument()++ fireEvent.click(tableTrigger)+ expect(await screen.findByText('Partition by')).toBeInTheDocument()+ expect(screen.queryByRole('button', { name: 'Clear' })).not.toBeInTheDocument()++ fireEvent.click(tableTrigger)+ expect(screen.queryByText('Partition by')).not.toBeInTheDocument()+ expect(screen.getByText('Not configured')).toBeInTheDocument()+ })++ it('keeps the form pristine when a table row is only expanded', async () => {+ mockSources()+ mockPublication()++ customRender(<TableOptionsHarness tableOptions={[]} />)++ const tableTrigger = await screen.findByRole('button', {+ name: 'public.Orders: Not configured',+ })+ expect(screen.getByText('Form is pristine')).toBeInTheDocument()++ fireEvent.click(tableTrigger)+ expect(await screen.findByText('Partition by')).toBeInTheDocument()+ expect(screen.getByText('Form is pristine')).toBeInTheDocument()++ fireEvent.click(tableTrigger)+ expect(screen.getByText('Form is pristine')).toBeInTheDocument()+ })++ it('clears a configured layout without removing its table row', async () => {+ mockSources()+ mockPublication()+ mockColumns(101, [{ name: 'region', type: 'text', nullable: false, primary_key: false }])++ customRender(<TableOptionsHarness tableOptions={[{ tableId: 101, clusterBy: ['region'] }]} />)++ fireEvent.click(+ await screen.findByRole('button', { name: 'public.Orders: 1 clustering column' })+ )+ fireEvent.click(await screen.findByRole('button', { name: 'Clear' }))++ expect(+ screen.getByRole('button', { name: 'public.Orders: Not configured' })+ ).toBeInTheDocument()+ expect(screen.queryByText('Partition by')).not.toBeInTheDocument()+ })++ it('shows integer range validation errors beside the invalid fields', async () => {+ mockSources()+ mockPublication()+ mockColumns(101, [{ name: 'id', type: 'int8', nullable: false, primary_key: true }])++ customRender(+ <TableOptionsHarness+ tableOptions={[+ {+ tableId: 101,+ partitionBy: {+ kind: 'integer_range',+ column: 'id',+ start: 0,+ end: 0,+ interval: 0,+ },+ },+ ]}+ />+ )++ fireEvent.click(await screen.findByRole('button', { name: /public\.Orders: Integer range/ }))+ fireEvent.click(screen.getByRole('button', { name: 'Validate form' }))+ expect(await screen.findByText('Interval must be greater than 0.')).toBeInTheDocument()+ // Once beside the End field, and once on the trigger so a collapsed row still explains itself.+ expect(screen.getAllByText('End must be greater than start.')).toHaveLength(2)+ expect(+ screen.getByRole('button', { name: 'public.Orders: End must be greater than start.' })+ ).toBeInTheDocument()+ })++ it('blocks an incomplete partition instead of dropping it silently on save', async () => {+ mockSources()+ mockPublication()+ mockColumns(101, [+ { name: 'CreatedAt', type: 'timestamptz', nullable: false, primary_key: true },+ ])++ customRender(+ <TableOptionsHarness+ tableOptions={[{ tableId: 101, partitionBy: { kind: 'time_column', column: '' } }]}+ />+ )++ const tableTrigger = await screen.findByRole('button', {+ name: 'public.Orders: Time column partitioning',+ })++ fireEvent.click(screen.getByRole('button', { name: 'Validate form' }))+ expect(await screen.findByText('Select a partition column')).toBeInTheDocument()++ // The row can be collapsed when the save is attempted, so the trigger has to explain it too.+ expect(+ await screen.findByRole('button', { name: 'public.Orders: Select a partition column' })+ ).toBe(tableTrigger)+ })++ it('marks configured columns that no longer exist while preserving valid names and types', async () => {+ mockSources()+ mockPublication()+ mockColumns(101, [+ { name: 'region', type: 'text', nullable: false, primary_key: false },+ { name: 'CreatedAt', type: 'timestamptz', nullable: false, primary_key: false },+ ])++ customRender(+ <TableOptionsHarness+ tableOptions={[+ {+ tableId: 101,+ partitionBy: { kind: 'time_column', column: 'DroppedAt', granularity: 'day' },+ clusterBy: ['region', 'LegacySegment'],+ },+ ]}+ />+ )++ fireEvent.click(+ await screen.findByRole('button', { name: /public\.Orders: Daily by DroppedAt/ })+ )+ expect(+ await screen.findByText('Some selected columns are no longer available')+ ).toBeInTheDocument()+ expect(screen.getAllByText('region')[0]).toBeInTheDocument()+ expect(screen.getByText('text')).toBeInTheDocument()+ expect(screen.getByText('DroppedAt').parentElement).toHaveClass('text-destructive-600')+ expect(screen.getByText('LegacySegment').parentElement).toHaveClass('text-destructive-600')+ })++ it('marks existing columns that are no longer included in the publication', async () => {+ mockSources()+ mockPublication({+ ...publication,+ config: {+ type: 'tables',+ tables: [+ {+ id: publicationTable.id,+ schema: publicationTable.schema,+ name: publicationTable.name,+ columns: ['region'],+ row_filter: null,+ },+ ],+ operations: ['insert'],+ publish_via_partition_root: false,+ },+ })+ mockColumns(101, [+ { name: 'region', type: 'text', nullable: false, primary_key: false },+ { name: 'PrivateNote', type: 'text', nullable: true, primary_key: false },+ ])++ customRender(+ <TableOptionsHarness tableOptions={[{ tableId: 101, clusterBy: ['PrivateNote'] }]} />+ )++ fireEvent.click(+ await screen.findByRole('button', { name: /public\.Orders: 1 clustering column/ })+ )+ expect(+ await screen.findByText('Some selected columns are no longer available')+ ).toBeInTheDocument()+ expect(screen.getByText('PrivateNote').parentElement).toHaveClass('text-destructive-600')+ })++ it('inherits explicit column filters through nested partition ancestors', async () => {+ mockSources()+ mockPublication({+ ...publication,+ config: {+ type: 'tables',+ tables: [+ {+ id: 10,+ schema: 'public',+ name: 'Events',+ columns: ['AllowedColumn'],+ row_filter: null,+ },+ ],+ operations: ['insert'],+ publish_via_partition_root: false,+ },+ tables: [+ {+ id: 12,+ schema: 'public',+ name: 'Events2026August',+ kind: 'table',+ partition_parent_id: 11,+ },+ ],+ })+ mockTables([+ {+ id: 10,+ schema: 'public',+ name: 'Events',+ kind: 'partitioned_table',+ partition_parent_id: null,+ },+ {+ id: 11,+ schema: 'public',+ name: 'Events2026',+ kind: 'partitioned_table',+ partition_parent_id: 10,+ },+ {+ id: 12,+ schema: 'public',+ name: 'Events2026August',+ kind: 'table',+ partition_parent_id: 11,+ },+ ])+ mockColumns(12, [+ { name: 'AllowedColumn', type: 'text', nullable: false, primary_key: false },+ { name: 'ExcludedColumn', type: 'text', nullable: false, primary_key: false },+ ])++ customRender(+ <TableOptionsHarness tableOptions={[{ tableId: 12, clusterBy: ['ExcludedColumn'] }]} />+ )++ fireEvent.click(+ await screen.findByRole('button', {+ name: /public\.Events2026August: 1 clustering column/,+ })+ )+ expect(+ await screen.findByText('Some selected columns are no longer available')+ ).toBeInTheDocument()+ expect(screen.getByText('ExcludedColumn').parentElement).toHaveClass('text-destructive-600')+ })++ it('does not treat an unresolved partition ancestry as publishing every column', async () => {+ mockSources()+ mockPublication({+ ...publication,+ config: {+ type: 'tables',+ tables: [+ {+ id: 10,+ schema: 'public',+ name: 'Events',+ columns: ['AllowedColumn'],+ row_filter: null,+ },+ ],+ operations: ['insert'],+ publish_via_partition_root: false,+ },+ tables: [+ {+ id: 12,+ schema: 'public',+ name: 'Events2026August',+ kind: 'table',+ partition_parent_id: 11,+ },+ ],+ })+ mockTables([+ {+ id: 12,+ schema: 'public',+ name: 'Events2026August',+ kind: 'table',+ partition_parent_id: 11,+ },+ ])+ mockColumns(12, [+ { name: 'AllowedColumn', type: 'text', nullable: false, primary_key: false },+ { name: 'UnverifiedColumn', type: 'text', nullable: false, primary_key: false },+ ])++ customRender(+ <TableOptionsHarness tableOptions={[{ tableId: 12, clusterBy: ['UnverifiedColumn'] }]} />+ )++ fireEvent.click(+ await screen.findByRole('button', {+ name: /public\.Events2026August: 1 clustering column/,+ })+ )+ expect(await screen.findByText('Columns could not be verified')).toBeInTheDocument()+ expect(+ screen.queryByText('Some selected columns are no longer available')+ ).not.toBeInTheDocument()+ })++ it('hides settings for a table removed from the publication without exposing its id', async () => {+ mockSources()+ mockPublication()+ mockTables([+ publicationTable,+ {+ id: 202,+ schema: 'Billing',+ name: 'Invoices',+ kind: 'table',+ partition_parent_id: null,+ },+ ])+ customRender(<TableOptionsHarness tableOptions={[{ tableId: 202, clusterBy: ['Region'] }]} />)++ expect(await screen.findByText('Billing.Invoices')).toBeInTheDocument()+ expect(screen.getByText('No longer in publication')).toHaveClass('text-destructive-600')+ expect(screen.getByText('Some tables are no longer in the publication')).toBeInTheDocument()+ expect(screen.queryByText('Region')).not.toBeInTheDocument()+ expect(document.body).not.toHaveTextContent('202')+ })++ it('does not match a dropped and recreated table by name instead of id', async () => {+ mockSources()+ mockPublication()+ mockTables([+ publicationTable,+ {+ id: 1000,+ schema: 'public',+ name: 'PreviouslyConfigured',+ kind: 'table',+ partition_parent_id: null,+ },+ ])++ customRender(+ <TableOptionsHarness+ tableOptions={[+ {+ tableId: 999,+ partitionBy: { kind: 'time_column', column: 'OccurredAt', granularity: 'day' },+ },+ ]}+ />+ )++ expect(await screen.findByText('Previously configured table')).toBeInTheDocument()+ expect(screen.queryByText('OccurredAt')).not.toBeInTheDocument()+ expect(screen.getByText('Some tables are no longer in the publication')).toBeInTheDocument()+ expect(document.body).not.toHaveTextContent('999')+ })+})diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.tsxnew file mode 100644index 00000000000..13316f87ab9--- /dev/null+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.tsx@@ -0,0 +1,304 @@+import { useParams } from 'common'+import { Table2, Trash2 } from 'lucide-react'+import { useState } from 'react'+import { useFieldArray, useFormState, useWatch, type Control } from 'react-hook-form'+import { Accordion, AccordionContent, AccordionItem, AccordionTrigger, Button, cn } from 'ui'+import { Admonition } from 'ui-patterns/Admonition'+import { ShimmeringLoader } from 'ui-patterns/ShimmeringLoader'++import type { DestinationPanelSchemaType } from '../DestinationForm.schema'+import { isMetadataValueLoading } from '../useRefreshOnOpen'+import { TableOptionDraftRow, TableOptionRow } from './TableOptionRow'+import { findFirstErrorMessage, resolvePublishedColumnNames } from './TableOptions.utils'+import { useReplicationPublicationQuery } from '@/data/replication/publication-query'+import { useReplicationSourceId } from '@/data/replication/sources-query'+import { useReplicationTablesQuery } from '@/data/replication/tables-query'++const tableLabel = (table: { schema: string; name: string }) => `${table.schema}.${table.name}`++const GRANULARITY_SUMMARIES = {+ hour: 'Hourly',+ day: 'Daily',+ month: 'Monthly',+ year: 'Yearly',+} as const++// Returns undefined when a table has nothing applied, so the caller can both render and+// de-emphasise the placeholder rather than comparing against its wording.+const tableOptionSummary = (+ option: NonNullable<DestinationPanelSchemaType['tableOptions']>[number] | undefined+): string | undefined => {+ if (option === undefined) return undefined++ const parts: string[] = []+ const partitionBy = option.partitionBy++ if (partitionBy?.kind === 'time_column') {+ parts.push(+ partitionBy.column+ ? `${GRANULARITY_SUMMARIES[partitionBy.granularity ?? 'day']} by ${partitionBy.column}`+ : 'Time column partitioning'+ )+ } else if (partitionBy?.kind === 'integer_range') {+ parts.push(+ partitionBy.column ? `Integer range by ${partitionBy.column}` : 'Integer range partitioning'+ )+ } else if (partitionBy?.kind === 'ingestion_time') {+ parts.push(+ `${GRANULARITY_SUMMARIES[partitionBy.granularity ?? 'day']} ingestion-time partitioning`+ )+ }++ const clusteringColumnCount = option.clusterBy?.length ?? 0+ if (clusteringColumnCount > 0) {+ parts.push(+ `${clusteringColumnCount} clustering ${clusteringColumnCount === 1 ? 'column' : 'columns'}`+ )+ }++ return parts.length > 0 ? parts.join(' · ') : undefined+}++// Mirrors the row list so the list does not resize when the publication resolves.+const TableOptionsSkeleton = () => (+ <div className="overflow-hidden rounded-md border" aria-hidden="true">+ {['w-2/3', 'w-1/2', 'w-3/4'].map((width, index) => (+ <div key={width} className="flex flex-col gap-1.5 px-3.5 py-3.5 border-t first:border-t-0">+ <ShimmeringLoader className={cn('h-3 py-0', width)} delayIndex={index} />+ <ShimmeringLoader className="h-3 w-1/4 py-0" delayIndex={index} />+ </div>+ ))}+ </div>+)++interface TableOptionsProps {+ control: Control<DestinationPanelSchemaType>+}++export const TableOptions = ({ control }: TableOptionsProps) => {+ const { ref: projectRef } = useParams()+ const sourceId = useReplicationSourceId({ projectRef })+ const publicationName = useWatch({ control, name: 'publicationName' })+ const tableOptions = useWatch({ control, name: 'tableOptions' }) ?? []+ const [expandedTableId, setExpandedTableId] = useState<number>()++ const {+ data: selectedPublication,+ isPending,+ isFetching,+ isError: isPublicationError,+ isSuccess: isPublicationSuccess,+ } = useReplicationPublicationQuery({+ projectRef,+ sourceId,+ publicationName,+ })+ const isLoadingPublicationTables = isMetadataValueLoading(+ isPending || isFetching,+ selectedPublication+ )+ const publicationTables = [...(selectedPublication?.tables ?? [])].sort((a, b) =>+ tableLabel(a).localeCompare(tableLabel(b))+ )++ const { fields, append, remove } = useFieldArray({ control, name: 'tableOptions' })+ const fieldIndexByTableId = new Map(fields.map((field, index) => [field.tableId, index]))+ const { errors } = useFormState({ control, name: 'tableOptions' })+ const publicationTableIds = new Set(publicationTables.map(({ id }) => id))+ const publicationTablesById = new Map(publicationTables.map((table) => [table.id, table]))+ const configuredTables =+ selectedPublication?.config.type === 'tables' ? selectedPublication.config.tables : []+ const configuredTablesById = new Map(configuredTables.map((table) => [table.id, table]))+ const unavailableTableOptions = isPublicationSuccess+ ? fields+ .map((field, index) => ({ field, index }))+ .filter(({ field }) => !publicationTableIds.has(field.tableId))+ : []+ // An expanded row needs its published columns resolved even before it has an entry, so that+ // the first column picker it opens is already filtered correctly.+ const activeTableIds = new Set([+ ...fields.map(({ tableId }) => tableId),+ ...(expandedTableId === undefined ? [] : [expandedTableId]),+ ])+ const tablesNeedingPartitionAncestry = new Set(+ [...activeTableIds].flatMap((tableId) => {+ if (!publicationTableIds.has(tableId) || configuredTablesById.has(tableId)) return []++ const parentId = publicationTablesById.get(tableId)?.partition_parent_id+ return parentId !== null && parentId !== undefined && !configuredTablesById.has(parentId)+ ? [tableId]+ : []+ })+ )+ const shouldResolvePartitionAncestry =+ selectedPublication?.config.type === 'tables' && tablesNeedingPartitionAncestry.size > 0+ const { data: sourceTables = [], isPending: isSourceTablesPending } = useReplicationTablesQuery(+ { projectRef, sourceId },+ {+ enabled: unavailableTableOptions.length > 0 || shouldResolvePartitionAncestry,+ }+ )+ const sourceTablesById = new Map(sourceTables.map((table) => [table.id, table]))+ const knownSourceTablesById = new Map(+ [...publicationTables, ...sourceTables].map((table) => [table.id, table])+ )++ if (!publicationName) {+ return (+ <Admonition+ type="note"+ title="Select a publication"+ description="Choose the publication whose destination tables you want to partition or cluster."+ />+ )+ }++ if (isLoadingPublicationTables) {+ return <TableOptionsSkeleton />+ }++ if (isPublicationError) {+ return (+ <Admonition+ type="warning"+ title="Table settings could not be loaded"+ description="Refresh the page before changing or saving the table layout."+ />+ )+ }++ if (publicationTables.length === 0 && unavailableTableOptions.length === 0) {+ return <p className="text-sm text-foreground-lighter">This publication has no tables.</p>+ }++ return (+ <div className="flex flex-col gap-y-3">+ <div className="overflow-hidden rounded-md border">+ <Accordion+ type="single"+ collapsible+ value={expandedTableId?.toString() ?? ''}+ onValueChange={(value) => setExpandedTableId(value === '' ? undefined : Number(value))}+ >+ {publicationTables.map((table) => {+ const index = fieldIndexByTableId.get(table.id)+ const summary = tableOptionSummary(+ index === undefined ? undefined : tableOptions[index]+ )+ // Errors live inside the collapsed row, so the trigger has to carry them or a+ // blocked save looks like nothing happened at all.+ const rowError =+ index === undefined ? undefined : findFirstErrorMessage(errors.tableOptions?.[index])+ const summaryLabel = rowError ?? summary ?? 'Not configured'+ const publishedColumnNames =+ selectedPublication?.config.type === 'tables'+ ? resolvePublishedColumnNames(table.id, configuredTablesById, knownSourceTablesById)+ : undefined+ const isPublicationColumnsPending =+ tablesNeedingPartitionAncestry.has(table.id) && isSourceTablesPending+ const sharedProps = {+ isPublicationColumnsError:+ selectedPublication?.config.type === 'tables' &&+ publishedColumnNames === null &&+ !isPublicationColumnsPending,+ isPublicationColumnsPending,+ publishedColumnNames,+ tableId: table.id,+ }++ return (+ <AccordionItem key={table.id} value={table.id.toString()} className="last:border-b-0">+ <AccordionTrigger+ aria-label={`${tableLabel(table)}: ${summaryLabel}`}+ className="px-3.5 py-3 font-normal hover:bg-surface-200 hover:no-underline [&>svg]:text-foreground-lighter"+ >+ <span className="flex min-w-0 items-center gap-3.5">+ <Table2+ size={16}+ strokeWidth={1.5}+ className="shrink-0 text-foreground-lighter"+ />+ <span className="flex min-w-0 flex-col gap-0.5">+ <span className="truncate text-sm font-medium text-foreground">+ {tableLabel(table)}+ </span>+ <span+ className={cn(+ 'truncate text-sm',+ rowError && 'text-destructive-600',+ !rowError && summary && 'text-foreground-lighter',+ // Most rows in a large publication are unconfigured, so keep the+ // placeholder quieter than the summaries worth scanning for.+ !rowError && !summary && 'text-foreground-muted'+ )}+ >+ {summaryLabel}+ </span>+ </span>+ </span>+ </AccordionTrigger>+ {/* Radix unmounts closed content after its exit animation, so this must not be+ gated on the expanded state as well or the row collapses as an empty box. */}+ <AccordionContent className="[&>div]:p-0">+ {index !== undefined ? (+ <TableOptionRow+ control={control}+ index={index}+ onClear={() => {+ remove(index)+ setExpandedTableId(undefined)+ }}+ {...sharedProps}+ />+ ) : (+ <TableOptionDraftRow+ onCreate={(option) => append({ tableId: table.id, ...option })}+ {...sharedProps}+ />+ )}+ </AccordionContent>+ </AccordionItem>+ )+ })}+ </Accordion>++ {unavailableTableOptions.map(({ field, index }) => {+ const sourceTable = sourceTablesById.get(field.tableId)++ return (+ <div+ key={field.id}+ className="flex items-center justify-between gap-3 border-t px-3.5 py-3"+ >+ <span className="flex min-w-0 items-center gap-3.5">+ <Table2 size={16} strokeWidth={1.5} className="shrink-0 text-foreground-lighter" />+ <span className="flex min-w-0 flex-col gap-0.5">+ <span className="truncate text-sm font-medium text-foreground">+ {sourceTable ? tableLabel(sourceTable) : 'Previously configured table'}+ </span>+ <span className="text-sm text-destructive-600">No longer in publication</span>+ </span>+ </span>+ <Button+ type="button"+ variant="default"+ icon={<Trash2 size={14} />}+ onClick={() => remove(index)}+ >+ Remove+ </Button>+ </div>+ )+ })}+ </div>++ {unavailableTableOptions.length > 0 && (+ <Admonition+ type="destructive"+ title="Some tables are no longer in the publication"+ description="Remove their configuration before saving."+ />+ )}+ </div>+ )+}diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.utils.test.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.utils.test.tsnew file mode 100644index 00000000000..ca6c8415bf3--- /dev/null+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.utils.test.ts@@ -0,0 +1,116 @@+import { describe, expect, it } from 'vitest'++import {+ findFirstErrorMessage,+ isIntegerPartitionColumnType,+ isPartitionColumnTypeCompatible,+ isTimePartitionColumnType,+ resolvePublishedColumnNames,+} from './TableOptions.utils'++describe('resolvePublishedColumnNames', () => {+ it('returns the columns from the nearest configured partition ancestor', () => {+ const configuredTables = new Map([+ [1, { id: 1, columns: ['root_column'] }],+ [2, { id: 2, columns: ['middle_column'] }],+ ])+ const sourceTables = new Map([+ [1, { id: 1, partition_parent_id: null }],+ [2, { id: 2, partition_parent_id: 1 }],+ [3, { id: 3, partition_parent_id: 2 }],+ ])++ expect(resolvePublishedColumnNames(3, configuredTables, sourceTables)).toEqual(+ new Set(['middle_column'])+ )+ })++ it('returns undefined when the nearest configured ancestor publishes every column', () => {+ const configuredTables = new Map([[1, { id: 1, columns: null }]])+ const sourceTables = new Map([+ [1, { id: 1, partition_parent_id: null }],+ [2, { id: 2, partition_parent_id: 1 }],+ ])++ expect(resolvePublishedColumnNames(2, configuredTables, sourceTables)).toBeUndefined()+ })++ it('returns null when an ancestry link cannot be resolved', () => {+ expect(+ resolvePublishedColumnNames(+ 3,+ new Map([[1, { id: 1, columns: ['id'] }]]),+ new Map([[3, { id: 3, partition_parent_id: 2 }]])+ )+ ).toBeNull()+ })++ it('returns null for a cyclic partition ancestry', () => {+ const sourceTables = new Map([+ [2, { id: 2, partition_parent_id: 3 }],+ [3, { id: 3, partition_parent_id: 2 }],+ ])++ expect(resolvePublishedColumnNames(3, new Map(), sourceTables)).toBeNull()+ })++ it('returns null when no configured ancestor exists', () => {+ const sourceTables = new Map([[1, { id: 1, partition_parent_id: null }]])++ expect(resolvePublishedColumnNames(1, new Map(), sourceTables)).toBeNull()+ })+})++describe('partition column type compatibility', () => {+ it('accepts Postgres date and timestamp types for time-column partitioning', () => {+ expect(isTimePartitionColumnType('date')).toBe(true)+ expect(isTimePartitionColumnType('timestamp with time zone')).toBe(true)+ expect(isTimePartitionColumnType('timestamptz')).toBe(true)+ expect(isTimePartitionColumnType('integer')).toBe(false)+ })++ it('accepts integer family types for integer-range partitioning', () => {+ expect(isIntegerPartitionColumnType('bigint')).toBe(true)+ expect(isIntegerPartitionColumnType('int4')).toBe(true)+ expect(isIntegerPartitionColumnType('date')).toBe(false)+ })++ it('treats an unknown type as compatible so columns can still be chosen while loading', () => {+ expect(isPartitionColumnTypeCompatible('time_column')).toBe(true)+ expect(isPartitionColumnTypeCompatible('integer_range', 'text')).toBe(false)+ })+})++describe('findFirstErrorMessage', () => {+ it('returns the first message nested anywhere inside a field-array error', () => {+ expect(+ findFirstErrorMessage({+ partitionBy: { column: { type: 'too_small', message: 'Select a partition column' } },+ })+ ).toBe('Select a partition column')+ })++ it('ignores branches without a message rather than reporting an empty error', () => {+ expect(findFirstErrorMessage(undefined)).toBeUndefined()+ expect(+ findFirstErrorMessage({ partitionBy: { column: { type: 'too_small' } } })+ ).toBeUndefined()+ expect(findFirstErrorMessage({ clusterBy: { message: '' } })).toBeUndefined()+ })++ it('does not walk into the DOM node that react-hook-form attaches to every field error', () => {+ const input = document.createElement('input')+ input.setAttribute('message', 'not an error message')++ expect(+ findFirstErrorMessage({ partitionBy: { column: { type: 'too_small', ref: input } } })+ ).toBeUndefined()+ })++ it('survives a cyclic error graph', () => {+ const partitionBy: Record<string, unknown> = { column: { type: 'too_small' } }+ partitionBy.self = partitionBy++ expect(findFirstErrorMessage({ partitionBy })).toBeUndefined()+ })+})diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.utils.tsnew file mode 100644index 00000000000..200db448dbd--- /dev/null+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/BigQuery/TableOptions.utils.ts@@ -0,0 +1,110 @@+const TIME_PARTITION_COLUMN_TYPES = new Set([+ 'date',+ 'timestamp',+ 'timestamptz',+ 'timestamp with time zone',+ 'timestamp without time zone',+])++const INTEGER_PARTITION_COLUMN_TYPES = new Set([+ 'smallint',+ 'integer',+ 'bigint',+ 'int2',+ 'int4',+ 'int8',+ 'serial',+ 'serial2',+ 'serial4',+ 'serial8',+ 'smallserial',+ 'bigserial',+])++export const isTimePartitionColumnType = (type?: string) =>+ type !== undefined && TIME_PARTITION_COLUMN_TYPES.has(type.toLowerCase())++export const isIntegerPartitionColumnType = (type?: string) =>+ type !== undefined && INTEGER_PARTITION_COLUMN_TYPES.has(type.toLowerCase())++export const isPartitionColumnTypeCompatible = (+ kind: 'time_column' | 'integer_range',+ type?: string+) => {+ if (type === undefined) return true+ return kind === 'time_column'+ ? isTimePartitionColumnType(type)+ : isIntegerPartitionColumnType(type)+}++export type PublicationTableConfig = {+ id: number+ columns?: string[] | null+}++export type SourceTableWithPartitionParent = {+ id: number+ partition_parent_id?: number | null+}++// `undefined` means the nearest configured publication entry includes every column. `null`+// means the ancestry could not be resolved safely, so the UI must not claim any configured+// column is still valid.+export const resolvePublishedColumnNames = (+ tableId: number,+ configuredTablesById: ReadonlyMap<number, PublicationTableConfig>,+ sourceTablesById: ReadonlyMap<number, SourceTableWithPartitionParent>+): Set<string> | null | undefined => {+ const visitedTableIds = new Set<number>()+ let currentTableId: number | null | undefined = tableId++ while (currentTableId !== null && currentTableId !== undefined) {+ if (visitedTableIds.has(currentTableId)) return null+ visitedTableIds.add(currentTableId)++ const configuredTable = configuredTablesById.get(currentTableId)+ if (configuredTable) {+ return configuredTable.columns === null || configuredTable.columns === undefined+ ? undefined+ : new Set(configuredTable.columns)+ }++ const sourceTable = sourceTablesById.get(currentTableId)+ if (!sourceTable) return null+ currentTableId = sourceTable.partition_parent_id+ }++ return null+}++// React Hook Form nests field-array errors by field name. A collapsed row can only show one+// line, and any error there blocks the save, so the first message is enough to explain why.+// Only plain objects and arrays are worth walking. React Hook Form hangs a `ref` on every+// field error, which points at a DOM node whose graph is deep and cyclic.+const isTraversable = (value: unknown): value is Record<string, unknown> => {+ if (typeof value !== 'object' || value === null) return false+ if (Array.isArray(value)) return true++ const prototype = Object.getPrototypeOf(value)+ return prototype === Object.prototype || prototype === null+}++export const findFirstErrorMessage = (+ error: unknown,+ visited = new WeakSet<object>()+): string | undefined => {+ if (!isTraversable(error) || visited.has(error)) return undefined+ visited.add(error)++ const { message } = error+ if (typeof message === 'string' && message.length > 0) return message++ for (const [key, nested] of Object.entries(error)) {+ if (key === 'ref') continue++ const nestedMessage = findFirstErrorMessage(nested, visited)+ if (nestedMessage !== undefined) return nestedMessage+ }++ return undefined+}diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.tsindex 2b5b503f439..85d29ea363a 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts@@ -11,6 +11,7 @@ import {buildTableSyncCopyConfig,generateDefaultValues,pruneStaleSelectedTableIds,+ pruneStaleTableOptions,} from './DestinationForm.utils'import { getDucklakeValidationIssues } from './DuckLake/DuckLake.utils'import { getSnowflakeValidationIssues } from './Snowflake/Snowflake.utils'@@ -227,6 +228,50 @@ describe('DestinationForm.utils table copy selection', () => {})})+describe('pruneStaleTableOptions', () => {+ const publication: ReplicationPublicationData = {+ name: 'analytics',+ config: {+ type: 'tables',+ tables: [{ id: 101, schema: 'public', name: 'orders', columns: null, row_filter: null }],+ operations: ['insert'],+ publish_via_partition_root: false,+ },+ tables: [+ {+ id: 101,+ schema: 'public',+ name: 'orders',+ kind: 'table',+ partition_parent_id: null,+ },+ ],+ }++ it('drops table options whose ids are no longer in the publication', () => {+ expect(+ pruneStaleTableOptions({+ tableOptions: [+ { tableId: 101, clusterBy: ['region'] },+ { tableId: 202, clusterBy: ['unused'] },+ ],+ publication,+ publicationName: 'analytics',+ })+ ).toEqual([{ tableId: 101, clusterBy: ['region'] }])+ })++ it('returns undefined when table options were never set', () => {+ expect(+ pruneStaleTableOptions({+ tableOptions: undefined,+ publication,+ publicationName: 'analytics',+ })+ ).toBeUndefined()+ })+})+const baseClickHouseFormData = {name: 'ClickHouse Destination',publicationName: 'pub',diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.tsindex 4c7da8776cf..4f3fb97e1e2 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts@@ -30,6 +30,8 @@ import { type ReplicationPublicationData } from '@/data/replication/publication-import type {BatchConfig,BigQueryDestinationConfig,+ BigQueryPartitionBy,+ BigQueryTableOption,ClickHouseDestinationConfig,DestinationConfig,DucklakeDestinationConfig,@@ -119,6 +121,7 @@ export const generateDefaultValues = ({(config as { big_query?: { connection_pool_size?: number } } | undefined)?.big_query?.connection_pool_size ?? DEFAULT_CONNECTION_POOL_SIZE,maxStalenessMins: bigQueryConfig?.max_staleness_mins ?? undefined, // Default: null+ tableOptions: (bigQueryConfig?.table_options?.tables ?? []).map(parseBigQueryTableOption),// Analytics Bucket fieldswarehouseName: icebergConfig?.warehouse_name ?? '',namespace: icebergConfig?.namespace ?? '',@@ -234,6 +237,58 @@ export const pruneStaleSelectedTableIds = ({return selectedTableIds.filter((id) => publicationTableIds.has(id))}+export const pruneStaleTableOptions = ({+ tableOptions,+ publication,+ publicationName,+}: {+ tableOptions?: BigQueryTableOption[]+ publication: ReplicationPublicationData+ publicationName: string+}): BigQueryTableOption[] | undefined => {+ if (tableOptions === undefined) return undefined++ const publicationTableIds = getPublicationTableIds(publication, publicationName)+ return tableOptions.filter((option) => publicationTableIds.has(String(option.tableId)))+}++// Derived from the generated response type rather than hand-typed, so `granularity` stays the+// same literal union the API actually returns instead of a widened `string` needing a cast.+type BigQueryConfigResponse = Extract<+ NonNullable<ReplicationDestinationByIdData['config']>,+ { big_query: unknown }+>['big_query']+type BigQueryTableOptionResponse = NonNullable<+ NonNullable<BigQueryConfigResponse['table_options']>['tables']+>[number]+type BigQueryPartitionByResponse = NonNullable<BigQueryTableOptionResponse['partition_by']>++const parseBigQueryPartitionBy = (+ partitionBy: BigQueryPartitionByResponse+): BigQueryPartitionBy => {+ switch (partitionBy.kind) {+ case 'time_column':+ return {+ kind: 'time_column',+ column: partitionBy.column,+ granularity: partitionBy.granularity ?? 'day',+ }+ case 'integer_range':+ return { ...partitionBy }+ case 'ingestion_time':+ return {+ kind: 'ingestion_time',+ granularity: partitionBy.granularity ?? 'day',+ }+ }+}++const parseBigQueryTableOption = (option: BigQueryTableOptionResponse): BigQueryTableOption => ({+ tableId: option.table_id,+ partitionBy: option.partition_by ? parseBigQueryPartitionBy(option.partition_by) : undefined,+ clusterBy: option.cluster_by ?? [],+})+const buildBigQueryConfig = (data: z.infer<typeof DestinationPanelFormSchema>): BigQueryDestinationConfig => ({@@ -242,6 +297,7 @@ const buildBigQueryConfig = (serviceAccountKey: data.serviceAccountKey ?? '',connectionPoolSize: data.connectionPoolSize,maxStalenessMins: data.maxStalenessMins,+ tableOptions: data.tableOptions,})const buildSnowflakeConfig = (diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsxindex d8005623ba0..4b732594581 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx@@ -59,6 +59,10 @@ export const PublicationSelection = ({shouldDirty: true,shouldValidate: true,})+ form.setValue('tableOptions', [], {+ shouldDirty: true,+ shouldValidate: true,+ })}field.onChange(value)},diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsxindex 7f6202b1423..80d904bab70 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx@@ -34,6 +34,7 @@ import {buildTableSyncCopyConfig,generateDefaultValues,pruneStaleSelectedTableIds,+ pruneStaleTableOptions,} from './DestinationForm.utils'import { DestinationNameInput } from './DestinationNameInput'import { getDucklakeValidationIssues } from './DuckLake/DuckLake.utils'@@ -314,6 +315,11 @@ export const DestinationForm = ({publication: selectedPublication,publicationName: rawData.publicationName,}),+ tableOptions: pruneStaleTableOptions({+ tableOptions: rawData.tableOptions,+ publication: selectedPublication,+ publicationName: rawData.publicationName,+ }),}if (@@ -546,6 +552,10 @@ export const DestinationForm = ({shouldDirty: true,shouldValidate: true,})+ form.setValue('tableOptions', [], {+ shouldDirty: true,+ shouldValidate: true,+ })form.setValue('publicationName', newPublication, {shouldDirty: true,shouldValidate: true,diff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.test.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.test.tsindex d112d7cc56a..a37f8b8b8b9 100644--- a/apps/studio/data/replication/create-destination-pipeline-mutation.test.ts+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.test.ts@@ -39,7 +39,7 @@ describe('buildBigQueryApiConfig', () => {serviceAccountKey: '{}',}- it('maps the destination config to the API shape', () => {+ it('omits table_options when none are configured', () => {expect(buildBigQueryApiConfig(baseConfig)).toEqual({big_query: {project_id: 'my-project',@@ -47,10 +47,128 @@ describe('buildBigQueryApiConfig', () => {service_account_key: '{}',connection_pool_size: undefined,max_staleness_mins: undefined,+ table_options: undefined,},})})+ it('maps per-table partitioning and clustering to the API shape', () => {+ expect(+ buildBigQueryApiConfig({+ ...baseConfig,+ tableOptions: [+ {+ tableId: 16_408,+ partitionBy: { kind: 'time_column', column: 'created_at', granularity: 'day' },+ clusterBy: ['customer_id', 'region'],+ },+ {+ tableId: 16_409,+ partitionBy: {+ kind: 'integer_range',+ column: 'shard',+ start: 0,+ end: 100,+ interval: 10,+ },+ },+ ],+ })+ ).toMatchObject({+ big_query: {+ table_options: {+ tables: [+ {+ table_id: 16_408,+ partition_by: { kind: 'time_column', column: 'created_at', granularity: 'day' },+ cluster_by: ['customer_id', 'region'],+ },+ {+ table_id: 16_409,+ partition_by: {+ kind: 'integer_range',+ column: 'shard',+ start: 0,+ end: 100,+ interval: 10,+ },+ cluster_by: undefined,+ },+ ],+ },+ },+ })+ })++ it('omits a time-column partition that has no column selected', () => {+ expect(+ buildBigQueryApiConfig({+ ...baseConfig,+ tableOptions: [+ { tableId: 16_408, partitionBy: { kind: 'time_column', column: '', granularity: 'day' } },+ {+ tableId: 16_409,+ partitionBy: { kind: 'time_column', column: '', granularity: 'day' },+ clusterBy: ['customer_id'],+ },+ ],+ })+ ).toMatchObject({+ big_query: {+ table_options: {+ tables: [+ {+ table_id: 16_409,+ partition_by: undefined,+ cluster_by: ['customer_id'],+ },+ ],+ },+ },+ })+ })++ it('omits a table entry that has neither partitioning nor clustering set', () => {+ expect(+ buildBigQueryApiConfig({+ ...baseConfig,+ tableOptions: [+ { tableId: 16_408, partitionBy: undefined, clusterBy: [] },+ {+ tableId: 16_409,+ partitionBy: { kind: 'ingestion_time', granularity: 'day' },+ },+ ],+ })+ ).toMatchObject({+ big_query: {+ table_options: {+ tables: [+ {+ table_id: 16_409,+ partition_by: { kind: 'ingestion_time', granularity: 'day' },+ },+ ],+ },+ },+ })+ })++ it('sends table_options: null on update when every configured table is removed or blank', () => {+ expect(+ buildBigQueryUpdateApiConfig({+ ...baseConfig,+ tableOptions: [{ tableId: 16_408, clusterBy: [] }],+ })+ ).toMatchObject({ big_query: { table_options: null } })+ })++ it('sends table_options: null on update when every configured table is removed', () => {+ expect(buildBigQueryUpdateApiConfig({ ...baseConfig, tableOptions: [] })).toMatchObject({+ big_query: { table_options: null },+ })+ })+it('omits blank service_account_key on update, but not on create', () => {const config = { ...baseConfig, serviceAccountKey: '' }diff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.tsindex b583ab6fd77..7c03be4f426 100644--- a/apps/studio/data/replication/create-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.ts@@ -5,11 +5,17 @@ import { toast } from 'sonner'import { replicationKeys } from './keys'import type {BigQueryDestinationConfig,+ BigQueryTableOption,DestinationConfig,DucklakeDestinationConfig,PipelineConfig,} from './types'-import { buildPipelineApiConfig, isDucklakeSupabaseConfig } from './utils'+import {+ buildBigQueryTableOptionApiConfig,+ buildPipelineApiConfig,+ getConfiguredBigQueryTableOptions,+ isDucklakeSupabaseConfig,+} from './utils'import { handleError, post } from '@/data/fetchers'import type { ResponseError, UseCustomMutationOptions } from '@/types'@@ -20,6 +26,13 @@ type CreateDestinationApiConfig = CreateDestinationPipelineBody['destination_contype CreateBigQueryApiConfig = Extract<CreateDestinationApiConfig, { big_query: unknown }>type CreateDucklakeApiConfig = Extract<CreateDestinationApiConfig, { ducklake: unknown }>+const buildBigQueryTableOptionsApiConfig = (tableOptions: BigQueryTableOption[] | undefined) => {+ const configuredTableOptions = getConfiguredBigQueryTableOptions(tableOptions)++ if (tableOptions === undefined || configuredTableOptions.length === 0) return undefined+ return { tables: configuredTableOptions.map(buildBigQueryTableOptionApiConfig) }+}+// Maps the studio-side BigQuery config to the snake_case `{ big_query: ... }` payload accepted// by the platform API. Shared by the create and validate mutations.export function buildBigQueryApiConfig(config: BigQueryDestinationConfig): CreateBigQueryApiConfig {@@ -30,6 +43,7 @@ export function buildBigQueryApiConfig(config: BigQueryDestinationConfig): Creatservice_account_key: config.serviceAccountKey,connection_pool_size: config.connectionPoolSize,max_staleness_mins: config.maxStalenessMins,+ table_options: buildBigQueryTableOptionsApiConfig(config.tableOptions),},}}diff --git a/apps/studio/data/replication/keys.ts b/apps/studio/data/replication/keys.tsindex 172d161b336..e2625a78243 100644--- a/apps/studio/data/replication/keys.ts+++ b/apps/studio/data/replication/keys.ts@@ -26,6 +26,11 @@ export const replicationKeys = {] as const,tables: (projectRef: string | undefined, source_id: number | undefined) =>['projects', projectRef, 'sources', source_id, 'tables'] as const,+ tableColumns: (+ projectRef: string | undefined,+ source_id: number | undefined,+ tableId: number | undefined+ ) => ['projects', projectRef, 'sources', source_id, 'tables', tableId, 'columns'] as const,costEstimate: (projectRef: string | undefined,source_id: number | undefined,diff --git a/apps/studio/data/replication/table-columns-query.ts b/apps/studio/data/replication/table-columns-query.tsnew file mode 100644index 00000000000..f8980abef47--- /dev/null+++ b/apps/studio/data/replication/table-columns-query.ts@@ -0,0 +1,59 @@+import { queryOptions, useQuery } from '@tanstack/react-query'++import { REPLICATION_METADATA_FRESHNESS_MS } from './constants'+import { replicationKeys } from './keys'+import { get, handleError } from '@/data/fetchers'+import type { ResponseError, UseCustomQueryOptions } from '@/types'++type ReplicationTableColumnsParams = { projectRef?: string; sourceId?: number; tableId?: number }++async function fetchReplicationTableColumns(+ { projectRef, sourceId, tableId }: ReplicationTableColumnsParams,+ signal?: AbortSignal+) {+ if (!projectRef) throw new Error('projectRef is required')+ if (!sourceId) throw new Error('sourceId is required')+ if (tableId === undefined) throw new Error('tableId is required')++ const { data, error } = await get(+ '/platform/replication/v2/{ref}/sources/{source_id}/tables/{table_id}/columns',+ {+ params: { path: { ref: projectRef, source_id: sourceId, table_id: tableId } },+ signal,+ }+ )+ if (error) {+ handleError(error)+ }++ return data.columns+}++export type ReplicationTableColumnsData = Awaited<ReturnType<typeof fetchReplicationTableColumns>>++type ReplicationTableColumnsQueryOptions<TData> = Omit<+ UseCustomQueryOptions<ReplicationTableColumnsData, ResponseError, TData>,+ 'enabled'+> & { enabled?: boolean }++export const replicationTableColumnsQueryOptions = <TData = ReplicationTableColumnsData>(+ { projectRef, sourceId, tableId }: ReplicationTableColumnsParams,+ { enabled = true, ...options }: ReplicationTableColumnsQueryOptions<TData> = {}+) =>+ queryOptions<ReplicationTableColumnsData, ResponseError, TData>({+ queryKey: replicationKeys.tableColumns(projectRef, sourceId, tableId),+ queryFn: ({ signal }) =>+ fetchReplicationTableColumns({ projectRef, sourceId, tableId }, signal),+ enabled:+ enabled &&+ typeof projectRef !== 'undefined' &&+ typeof sourceId !== 'undefined' &&+ typeof tableId !== 'undefined',+ staleTime: REPLICATION_METADATA_FRESHNESS_MS,+ ...options,+ })++export const useReplicationTableColumnsQuery = <TData = ReplicationTableColumnsData>(+ { projectRef, sourceId, tableId }: ReplicationTableColumnsParams,+ options: ReplicationTableColumnsQueryOptions<TData> = {}+) => useQuery(replicationTableColumnsQueryOptions({ projectRef, sourceId, tableId }, options))diff --git a/apps/studio/data/replication/types.ts b/apps/studio/data/replication/types.tsindex 7b6d875de52..3d4bb8751a6 100644--- a/apps/studio/data/replication/types.ts+++ b/apps/studio/data/replication/types.ts@@ -64,6 +64,43 @@ export type BigQueryDestinationConfig = {serviceAccountKey: stringconnectionPoolSize?: numbermaxStalenessMins?: number+ tableOptions?: BigQueryTableOption[]+}++export const BIGQUERY_TIME_PARTITION_GRANULARITIES = ['hour', 'day', 'month', 'year'] as const+export const BIGQUERY_MAX_CLUSTERING_COLUMNS = 4++export type BigQueryTimePartitionGranularity =+ (typeof BIGQUERY_TIME_PARTITION_GRANULARITIES)[number]++export type BigQueryPartitionBy =+ | { kind: 'time_column'; column: string; granularity?: BigQueryTimePartitionGranularity }+ | {+ kind: 'integer_range'+ column: string+ start: number | ''+ end: number | ''+ interval: number | ''+ }+ | { kind: 'ingestion_time'; granularity?: BigQueryTimePartitionGranularity }++export type CompleteBigQueryPartitionBy =+ | Exclude<BigQueryPartitionBy, { kind: 'integer_range' }>+ | {+ kind: 'integer_range'+ column: string+ start: number+ end: number+ interval: number+ }++// A single source table's BigQuery partitioning/clustering configuration. `tableId` is the+// source Postgres table OID, stable across renames, matching the id used by the replication+// tables/columns endpoints.+export type BigQueryTableOption = {+ tableId: number+ partitionBy?: BigQueryPartitionBy+ clusterBy?: string[]}export type IcebergDestinationConfig = {diff --git a/apps/studio/data/replication/update-destination-pipeline-mutation.ts b/apps/studio/data/replication/update-destination-pipeline-mutation.tsindex 033a315584f..d54dd8a413c 100644--- a/apps/studio/data/replication/update-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/update-destination-pipeline-mutation.ts@@ -6,11 +6,17 @@ import { optionalSecret } from './destination-secret-utils'import { replicationKeys } from './keys'import type {BigQueryDestinationConfig,+ BigQueryTableOption,DestinationConfig,DucklakeDestinationConfig,PipelineConfig,} from './types'-import { buildPipelineApiConfig, isDucklakeSupabaseConfig } from './utils'+import {+ buildBigQueryTableOptionApiConfig,+ buildPipelineApiConfig,+ getConfiguredBigQueryTableOptions,+ isDucklakeSupabaseConfig,+} from './utils'import { handleError, post } from '@/data/fetchers'import type { ResponseError, UseCustomMutationOptions } from '@/types'@@ -21,6 +27,15 @@ type UpdateDestinationApiConfig = UpdateDestinationPipelineBody['destination_contype UpdateBigQueryApiConfig = Extract<UpdateDestinationApiConfig, { big_query: unknown }>type UpdateDucklakeApiConfig = Extract<UpdateDestinationApiConfig, { ducklake: unknown }>+const buildBigQueryTableOptionsUpdateApiConfig = (+ tableOptions: BigQueryTableOption[] | undefined+) => {+ const configuredTableOptions = getConfiguredBigQueryTableOptions(tableOptions)+ if (tableOptions === undefined) return undefined+ if (configuredTableOptions.length === 0) return null+ return { tables: configuredTableOptions.map(buildBigQueryTableOptionApiConfig) }+}+export function buildBigQueryUpdateApiConfig(config: BigQueryDestinationConfig): UpdateBigQueryApiConfig {@@ -31,6 +46,7 @@ export function buildBigQueryUpdateApiConfig(service_account_key: optionalSecret(config.serviceAccountKey),connection_pool_size: config.connectionPoolSize,max_staleness_mins: config.maxStalenessMins,+ table_options: buildBigQueryTableOptionsUpdateApiConfig(config.tableOptions),},}}diff --git a/apps/studio/data/replication/utils.ts b/apps/studio/data/replication/utils.tsindex 6ad7c043055..e5df9f2a7e7 100644--- a/apps/studio/data/replication/utils.ts+++ b/apps/studio/data/replication/utils.ts@@ -1,4 +1,7 @@import type {+ BigQueryPartitionBy,+ BigQueryTableOption,+ CompleteBigQueryPartitionBy,CreatePipelineApiConfig,DucklakeDestinationConfig,DucklakeSupabaseDestinationConfig,@@ -70,3 +73,55 @@ export const buildPipelineApiConfig = ({}: undefined,})++const hasClusteringColumns = (clusterBy: string[] | undefined) => (clusterBy?.length ?? 0) > 0++const isCompleteBigQueryPartition = (+ partitionBy: BigQueryPartitionBy | undefined+): partitionBy is CompleteBigQueryPartitionBy => {+ if (!partitionBy) return false+ if (partitionBy.kind === 'ingestion_time') return true+ if (!('column' in partitionBy) || partitionBy.column.trim().length === 0) return false+ if (partitionBy.kind !== 'integer_range') return true+ return (+ typeof partitionBy.start === 'number' &&+ typeof partitionBy.end === 'number' &&+ typeof partitionBy.interval === 'number'+ )+}++const buildBigQueryPartitionByApiConfig = (partitionBy: CompleteBigQueryPartitionBy) => {+ switch (partitionBy.kind) {+ case 'time_column':+ return {+ kind: partitionBy.kind,+ column: partitionBy.column,+ granularity: partitionBy.granularity,+ }+ case 'integer_range':+ return {+ kind: partitionBy.kind,+ column: partitionBy.column,+ start: partitionBy.start,+ end: partitionBy.end,+ interval: partitionBy.interval,+ }+ case 'ingestion_time':+ return { kind: partitionBy.kind, granularity: partitionBy.granularity }+ }+}++export const buildBigQueryTableOptionApiConfig = (option: BigQueryTableOption) => ({+ table_id: option.tableId,+ partition_by: isCompleteBigQueryPartition(option.partitionBy)+ ? buildBigQueryPartitionByApiConfig(option.partitionBy)+ : undefined,+ cluster_by: hasClusteringColumns(option.clusterBy) ? option.clusterBy : undefined,+})++const isBigQueryTableOptionConfigured = (option: BigQueryTableOption) =>+ isCompleteBigQueryPartition(option.partitionBy) || hasClusteringColumns(option.clusterBy)++export const getConfiguredBigQueryTableOptions = (+ tableOptions: BigQueryTableOption[] | undefined+) => (tableOptions ?? []).filter(isBigQueryTableOptionConfigured)