The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
import type {
BigQueryPartitionBy,
BigQueryTableOption,
CompleteBigQueryPartitionBy,
CreatePipelineApiConfig,
DucklakeDestinationConfig,
DucklakeSupabaseDestinationConfig,
PipelineConfig,
} from './types'
import { MAX_RETRY_FAILURE_COUNT } from '@/data/query-client'
import { ResponseError } from '@/types'
const isLocal =
process.env.NEXT_PUBLIC_ENVIRONMENT !== 'prod' &&
process.env.NEXT_PUBLIC_ENVIRONMENT !== 'staging'
export const checkLocalETLNotSetUp = (error: ResponseError | null) => {
if (error === null) return false
const isETLAPINotRunning =
error.code === undefined && error.message.includes('API error happened')
const isETLNotSetUp =
error.code === 503 && error.message.includes('replication API URL is not configured')
return isLocal && (isETLAPINotRunning || isETLNotSetUp)
}
export const checkReplicationFeatureFlagRetry = (
failureCount: number,
error: ResponseError
): boolean => {
const isFeatureFlagRequiredError =
error instanceof ResponseError &&
error.code === 503 &&
error.message.includes('feature flag is required')
const isLocalETLNotSetUp = checkLocalETLNotSetUp(error)
if (isFeatureFlagRequiredError || isLocalETLNotSetUp) {
return false
}
if (failureCount < MAX_RETRY_FAILURE_COUNT) {
return true
}
return false
}
export function isDucklakeSupabaseConfig(
config: DucklakeDestinationConfig
): config is DucklakeSupabaseDestinationConfig {
return 'catalogProjectRef' in config
}
export const buildPipelineApiConfig = ({
publicationName,
batch,
maxTableSyncWorkers,
maxCopyConnectionsPerTable,
invalidatedSlotBehavior,
tableSyncCopy,
}: PipelineConfig): CreatePipelineApiConfig => ({
publication_name: publicationName,
max_table_sync_workers: maxTableSyncWorkers,
max_copy_connections_per_table: maxCopyConnectionsPerTable,
invalidated_slot_behavior: invalidatedSlotBehavior,
table_sync_copy: tableSyncCopy,
batch: batch
? {
max_fill_ms: batch.maxFillMs,
max_bytes: batch.maxBytes,
memory_budget_ratio: batch.memoryBudgetRatio,
}
: undefined,
})
const hasClusteringColumns = (clusterBy: string[] | undefined) => (clusterBy?.length ?? 0) > 0
const isCompleteBigQueryPartition = (
partitionBy: BigQueryPartitionBy | undefined
): partitionBy is CompleteBigQueryPartitionBy => {
if (!partitionBy) return false
if (partitionBy.kind === 'ingestion_time') return true
if (!('column' in partitionBy) || partitionBy.column.trim().length === 0) return false
if (partitionBy.kind !== 'integer_range') return true
return (
typeof partitionBy.start === 'number' &&
typeof partitionBy.end === 'number' &&
typeof partitionBy.interval === 'number'
)
}
const buildBigQueryPartitionByApiConfig = (partitionBy: CompleteBigQueryPartitionBy) => {
switch (partitionBy.kind) {
case 'time_column':
return {
kind: partitionBy.kind,
column: partitionBy.column,
granularity: partitionBy.granularity,
}
case 'integer_range':
return {
kind: partitionBy.kind,
column: partitionBy.column,
start: partitionBy.start,
end: partitionBy.end,
interval: partitionBy.interval,
}
case 'ingestion_time':
return { kind: partitionBy.kind, granularity: partitionBy.granularity }
}
}
export const buildBigQueryTableOptionApiConfig = (option: BigQueryTableOption) => ({
table_id: option.tableId,
partition_by: isCompleteBigQueryPartition(option.partitionBy)
? buildBigQueryPartitionByApiConfig(option.partitionBy)
: undefined,
cluster_by: hasClusteringColumns(option.clusterBy) ? option.clusterBy : undefined,
})
const isBigQueryTableOptionConfigured = (option: BigQueryTableOption) =>
isCompleteBigQueryPartition(option.partitionBy) || hasClusteringColumns(option.clusterBy)
export const getConfiguredBigQueryTableOptions = (
tableOptions: BigQueryTableOption[] | undefined
) => (tableOptions ?? []).filter(isBigQueryTableOptionConfigured)