Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

fix(studio): add option to treat empty CSV cells as NULL on import (#43281)

Usama Nadeem55e0b34a18b5
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SidePanelEditor.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SidePanelEditor.tsx
index 01adfe7e077..2f2fb14b2e1 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SidePanelEditor.tsx
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SidePanelEditor.tsx
@@ -27,6 +27,7 @@ import { useSelectedOrganizationQuery } from 'hooks/misc/useSelectedOrganization
import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'
import { useConfirmOnClose } from 'hooks/ui/useConfirmOnClose'
import { useUrlState } from 'hooks/ui/useUrlState'
+import { useVisibleKey } from 'hooks/ui/useVisibleKey'
import { useTrack } from 'lib/telemetry/track'
import { isEmpty, isUndefined, noop } from 'lodash'
import { useState } from 'react'
@@ -66,6 +67,7 @@ import type { ImportContent } from './TableEditor/TableEditor.types'
import { useTableApiAccessPrivilegesMutation } from '@/data/privileges/table-api-access-mutation'
import { useDataApiGrantTogglesEnabled } from '@/hooks/misc/useDataApiGrantTogglesEnabled'
import { type ApiPrivilegesByRole } from '@/lib/data-api-types'
+import { isObjectContainingKeys } from '@/lib/helpers'
import type { DeepReadonly, Prettify } from '@/lib/type-helpers'
export type SaveTableParams =
@@ -201,6 +203,7 @@ export const SidePanelEditor = ({
const { updateRow, addRow, isEditPending } = useTableRowOperations()
const [isEdited, setIsEdited] = useState<boolean>(false)
+ const csvImportKey = useVisibleKey(snap.sidePanel?.type === 'csv-import')
const { data: publications } = useDatabasePublicationsQuery({
projectRef: project?.ref,
@@ -845,14 +848,13 @@ export const SidePanelEditor = ({
)
if (file && rowCount > 0) {
- // CSV file upload
- const res: any = await insertRowsViaSpreadsheet(
- project.ref!,
- project.connectionString,
+ const res = await insertRowsViaSpreadsheet({
+ projectRef: project.ref!,
+ connectionString: project.connectionString,
file,
- selectedTable,
+ table: selectedTable,
selectedHeaders,
- (progress: number) => {
+ onProgressUpdate: (progress: number) => {
toast.loading(
<SonnerProgress
progress={progress}
@@ -860,21 +862,24 @@ export const SidePanelEditor = ({
/>,
{ id: toastId }
)
- }
- )
+ },
+ treatEmptyAsNull: importContent.treatEmptyAsNull,
+ })
if (res.error) {
- toast.error(`Failed to import data: ${res.error.message}`, { id: toastId })
+ const message = isObjectContainingKeys(res.error, ['message'])
+ ? res.error.message
+ : 'An unknown error occurred during import'
+ toast.error(`Failed to import data: ${message}`, { id: toastId })
return resolve()
}
} else {
- // Text paste
- const res: any = await insertTableRows(
- project.ref!,
- project.connectionString,
- selectedTable,
- importContent.rows,
+ const res = await insertTableRows({
+ projectRef: project.ref!,
+ connectionString: project.connectionString,
+ table: selectedTable,
+ rows: importContent.rows,
selectedHeaders,
- (progress: number) => {
+ onProgressUpdate: (progress: number) => {
toast.loading(
<SonnerProgress
progress={progress}
@@ -884,10 +889,14 @@ export const SidePanelEditor = ({
/>,
{ id: toastId }
)
- }
- )
+ },
+ treatEmptyAsNull: importContent.treatEmptyAsNull,
+ })
if (res.error) {
- toast.error(`Failed to import data: ${res.error.message}`, { id: toastId })
+ const message = isObjectContainingKeys(res.error, ['message'])
+ ? res.error.message
+ : 'An unknown error occurred during import'
+ toast.error(`Failed to import data: ${message}`, { id: toastId })
return resolve()
}
}
@@ -990,6 +999,7 @@ export const SidePanelEditor = ({
onSelect={onSaveForeignRow}
/>
<SpreadsheetImport
+ key={csvImportKey}
visible={snap.sidePanel?.type === 'csv-import'}
selectedTable={selectedTable}
saveContent={onImportData}
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SidePanelEditor.utils.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SidePanelEditor.utils.tsx
index 035353306ff..9edb7cfa222 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SidePanelEditor.utils.tsx
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SidePanelEditor.utils.tsx
@@ -45,7 +45,7 @@ import {
} from 'data/tables/table-update-mutation'
import { getTables } from 'data/tables/tables-query'
import { sendEvent } from 'data/telemetry/send-event-mutation'
-import { timeout, tryParseJson } from 'lib/helpers'
+import { isObject, isObjectContainingKeys, timeout, tryParseJson } from 'lib/helpers'
import { chunk, find, isEmpty, isEqual } from 'lodash'
import Papa from 'papaparse'
import { toast } from 'sonner'
@@ -656,13 +656,13 @@ export const createTable = async ({
if (importContent.file && importContent.rowCount > 0) {
// Via a CSV file
- const { error }: any = await insertRowsViaSpreadsheet(
+ const { error } = await insertRowsViaSpreadsheet({
projectRef,
connectionString,
- importContent.file,
+ file: importContent.file,
table,
- importContent.selectedHeaders,
- (progress: number) => {
+ selectedHeaders: importContent.selectedHeaders,
+ onProgressUpdate: (progress: number) => {
toast.loading(
<div className="flex flex-col space-y-2" style={{ minWidth: '220px' }}>
<SparkBar
@@ -678,25 +678,28 @@ export const createTable = async ({
</div>,
{ id: toastId }
)
- }
- )
+ },
+ treatEmptyAsNull: importContent.treatEmptyAsNull,
+ })
if (error !== undefined) {
span.setAttribute('import.error', 1)
toast.error('Do check your spreadsheet if there are any discrepancies.')
- const message = `Table ${table.name} has been created but we ran into an error while inserting rows: ${error.message}`
+ const message = isObjectContainingKeys(error, ['message'])
+ ? String(error.message)
+ : 'An unknown error occurred during data import.'
toast.error(message)
console.error('Error:', { error, message })
}
} else {
// Via text copy and paste
- await insertTableRows(
+ await insertTableRows({
projectRef,
connectionString,
table,
- importContent.rows,
- importContent.selectedHeaders,
- (progress: number) => {
+ rows: importContent.rows,
+ selectedHeaders: importContent.selectedHeaders,
+ onProgressUpdate: (progress: number) => {
toast.loading(
<div className="flex flex-col space-y-2" style={{ minWidth: '220px' }}>
<SparkBar
@@ -711,8 +714,9 @@ export const createTable = async ({
</div>,
{ id: toastId }
)
- }
- )
+ },
+ treatEmptyAsNull: importContent.treatEmptyAsNull,
+ })
}
// For identity columns, manually raise the sequences (batched for performance)
@@ -966,18 +970,27 @@ export const formatRowsForInsert = ({
rows,
headers,
columns = [],
+ treatEmptyAsNull = true,
}: {
- rows: any[]
+ rows: unknown[]
headers: string[]
columns?: RetrieveTableResult['columns']
+ treatEmptyAsNull?: boolean
}) => {
- return rows.map((row: any) => {
- const formattedRow: any = {}
+ return rows.map((row) => {
+ const formattedRow: Record<string, unknown> = {}
+ if (!isObject(row)) {
+ return formattedRow
+ }
+
headers.forEach((header) => {
const column = columns?.find((c) => c.name === header)
+
const originalValue = row[header]
- if ((column?.format ?? '').includes('json')) {
+ if (originalValue === '' && treatEmptyAsNull && column?.is_nullable) {
+ formattedRow[header] = null
+ } else if ((column?.format ?? '').includes('json')) {
formattedRow[header] = tryParseJson(originalValue)
} else if ((column?.data_type ?? '') === 'ARRAY') {
if (
@@ -991,7 +1004,7 @@ export const formatRowsForInsert = ({
formattedRow[header] = tryParseJson(originalValue)
}
} else if (originalValue === '') {
- formattedRow[header] = column?.is_nullable ? null : ''
+ formattedRow[header] = ''
} else {
formattedRow[header] = originalValue
}
@@ -1000,17 +1013,26 @@ export const formatRowsForInsert = ({
})
}
-export const insertRowsViaSpreadsheet = async (
- projectRef: string,
- connectionString: string | undefined | null,
- file: any,
- table: RetrieveTableResult,
- selectedHeaders: string[],
+export async function insertRowsViaSpreadsheet({
+ projectRef,
+ connectionString,
+ file,
+ table,
+ selectedHeaders,
+ onProgressUpdate,
+ treatEmptyAsNull = true,
+}: {
+ projectRef: string
+ connectionString: string | undefined | null
+ file: File
+ table: RetrieveTableResult
+ selectedHeaders: string[]
onProgressUpdate: (progress: number) => void
-) => {
+ treatEmptyAsNull: boolean
+}): Promise<{ error: unknown }> {
let chunkNumber = 0
- let insertError: any = undefined
- const t1: any = new Date()
+ let insertError: unknown = undefined
+ const t1 = new Date()
return new Promise((resolve) => {
Papa.parse(file, {
header: true,
@@ -1019,13 +1041,14 @@ export const insertRowsViaSpreadsheet = async (
skipEmptyLines: true,
chunkSize: CHUNK_SIZE,
quoteChar: file.type === 'text/tab-separated-values' ? '' : '"',
- chunk: async (results: any, parser: any) => {
+ chunk: async (results, parser) => {
parser.pause()
const formattedData = formatRowsForInsert({
rows: results.data,
headers: selectedHeaders,
columns: table.columns,
+ treatEmptyAsNull,
})
const insertQuery = new Query().from(table.name, table.schema).insert(formattedData).toSql()
@@ -1046,33 +1069,45 @@ export const insertRowsViaSpreadsheet = async (
parser.resume()
},
complete: () => {
- const t2: any = new Date()
- console.log(`Total time taken for importing spreadsheet: ${(t2 - t1) / 1000} seconds`)
+ const t2 = new Date()
+ console.log(
+ `Total time taken for importing spreadsheet: ${(t2.getTime() - t1.getTime()) / 1000} seconds`
+ )
resolve({ error: insertError })
},
})
})
}
-export const insertTableRows = async (
- projectRef: string,
- connectionString: string | undefined | null,
- table: RetrieveTableResult,
- rows: any,
- selectedHeaders: string[],
+export async function insertTableRows({
+ projectRef,
+ connectionString,
+ table,
+ rows,
+ selectedHeaders,
+ onProgressUpdate,
+ treatEmptyAsNull = true,
+}: {
+ projectRef: string
+ connectionString: string | undefined | null
+ table: RetrieveTableResult
+ rows: unknown[]
+ selectedHeaders: string[]
onProgressUpdate: (progress: number) => void
-) => {
- let insertError = undefined
+ treatEmptyAsNull: boolean
+}): Promise<{ error: unknown }> {
+ let insertError: unknown = undefined
let insertProgress = 0
const formattedRows = formatRowsForInsert({
rows,
headers: selectedHeaders,
columns: table.columns,
+ treatEmptyAsNull,
})
const batches = chunk(formattedRows, BATCH_SIZE)
- const promises = batches.map((batch: any) => {
+ const tasks = batches.map((batch) => {
return () => {
return Promise.race([
new Promise(async (resolve, reject) => {
@@ -1087,12 +1122,12 @@ export const insertTableRows = async (
insertProgress = insertProgress + batch.length / rows.length
resolve({})
}),
- timeout(30000),
+ timeout(30_000),
])
}
})
- const batchedPromises = chunk(promises, 10)
+ const batchedPromises = chunk(tasks, 10)
for (const batchedPromise of batchedPromises) {
const res = await Promise.allSettled(batchedPromise.map((batch) => batch()))
const hasFailedBatch = find(res, { status: 'rejected' })
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadSheetFileUpload.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadSheetFileUpload.tsx
index 0914eb8e754..ff91de5f50a 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadSheetFileUpload.tsx
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadSheetFileUpload.tsx
@@ -1,25 +1,73 @@
-import { FileText, Loader } from 'lucide-react'
-import { DragEvent, useRef, useState } from 'react'
-
import SparkBar from 'components/ui/SparkBar'
-import { Button } from 'ui'
+import { FileText, Loader } from 'lucide-react'
+import { DragEvent, useCallback, useRef, useState, type ChangeEvent } from 'react'
+import { Button, cn } from 'ui'
interface SpreadSheetFileUploadProps {
parseProgress: number
- uploadedFile: any
- onFileUpload: (event: any) => void
+ uploadedFile: File | undefined
+ onFileUpload: (event: DragEvent<HTMLDivElement> | ChangeEvent<HTMLInputElement>) => void
removeUploadedFile: () => void
}
-const SpreadSheetFileUpload = ({
- parseProgress,
+export const SpreadSheetFileUpload = ({
uploadedFile,
onFileUpload,
+ parseProgress,
removeUploadedFile,
}: SpreadSheetFileUploadProps) => {
- const [isDraggedOver, setIsDraggedOver] = useState(false)
- const uploadButtonRef = useRef(null)
+ const uploadButtonRef = useRef<HTMLInputElement>(null)
+
+ const handleRemoveFile = useCallback(
+ function handleRemoveFile() {
+ removeUploadedFile()
+ if (uploadButtonRef.current) {
+ uploadButtonRef.current.value = ''
+ }
+ },
+ [removeUploadedFile]
+ )
+
+ return (
+ <div className="space-y-4">
+ <UploadInstructions />
+ {!uploadedFile ? (
+ <DropZone onDrop={onFileUpload} onClickUpload={() => uploadButtonRef.current?.click()} />
+ ) : (
+ <FileDetails
+ file={uploadedFile}
+ parseProgress={parseProgress}
+ removeFile={handleRemoveFile}
+ />
+ )}
+ <input ref={uploadButtonRef} className="hidden" type="file" onChange={onFileUpload} />
+ </div>
+ )
+}
+
+function UploadInstructions() {
+ return (
+ <div>
+ <p className="mb-2 text-sm text-foreground-light">
+ Upload a CSV or TSV file. The first row should be the headers of the table, and your headers
+ should not include any special characters other than hyphens (
+ <span className="text-code">-</span>) or underscores (<span className="text-code">_</span>
+ ).
+ </p>
+ <p className="text-sm text-foreground-light">
+ Tip: Datetime columns should be formatted as YYYY-MM-DD HH:mm:ss
+ </p>
+ </div>
+ )
+}
+
+interface DropZoneProps {
+ onDrop?: (event: DragEvent<HTMLDivElement>) => void
+ onClickUpload: () => void
+}
+function DropZone({ onDrop: onDropFromParent, onClickUpload }: DropZoneProps) {
+ const [isDraggedOver, setIsDraggedOver] = useState(false)
const onDragOver = (event: DragEvent<HTMLDivElement>) => {
if (event.type === 'dragover' && !isDraggedOver) {
setIsDraggedOver(true)
@@ -29,67 +77,66 @@ const SpreadSheetFileUpload = ({
event.stopPropagation()
event.preventDefault()
}
-
const onDrop = (event: DragEvent<HTMLDivElement>) => {
onDragOver(event)
- onFileUpload(event)
+ onDropFromParent?.(event)
}
return (
- <div className="space-y-4">
- <div>
- <p className="mb-2 text-sm text-foreground-light">
- Upload a CSV or TSV file. The first row should be the headers of the table, and your
- headers should not include any special characters other than hyphens (
- <span className="text-code">-</span>) or underscores (<span className="text-code">_</span>
- ).
- </p>
- <p className="text-sm text-foreground-light">
- Tip: Datetime columns should be formatted as YYYY-MM-DD HH:mm:ss
- </p>
+ <div
+ className={cn(
+ 'flex h-48 items-center justify-center',
+ 'rounded-md border border-dashed border-strong',
+ 'cursor-pointer',
+ isDraggedOver && 'bg-gray-500'
+ )}
+ onDragOver={onDragOver}
+ onDragLeave={onDragOver}
+ onDrop={onDrop}
+ onClick={onClickUpload}
+ >
+ <p className="text-sm">
+ Drag and drop, or <span className="text-brand">browse</span> your files
+ </p>
+ </div>
+ )
+}
+
+interface FileDetailsProps {
+ file: File
+ parseProgress: number
+ removeFile: () => void
+}
+
+function FileDetails({ file, parseProgress, removeFile }: FileDetailsProps) {
+ return (
+ <div
+ className={cn(
+ 'flex h-28 flex-col items-center justify-center space-y-2',
+ 'rounded-md border border-dashed border-strong'
+ )}
+ >
+ <div className="flex items-center space-x-2">
+ <FileText size={14} strokeWidth={1.5} />
+ <p className="text-sm text-foreground">{file.name}</p>
</div>
- {!uploadedFile ? (
- <div
- className={`flex h-48 cursor-pointer items-center justify-center rounded-md border border-dashed border-strong ${
- isDraggedOver ? 'bg-gray-500' : ''
- }`}
- onDragOver={onDragOver}
- onDragLeave={onDragOver}
- onDrop={onDrop}
- onClick={() => (uploadButtonRef.current as any)?.click()}
- >
- <p className="text-sm">
- Drag and drop, or <span className="text-brand">browse</span> your files
- </p>
- </div>
+ {parseProgress === 100 ? (
+ <Button type="outline" onClick={removeFile}>
+ Remove File
+ </Button>
) : (
- <div className="flex h-28 flex-col items-center justify-center space-y-2 rounded-md border border-dashed border-strong">
- <div className="flex items-center space-x-2">
- <FileText size={14} strokeWidth={1.5} />
- <p className="text-sm text-foreground">{uploadedFile.name}</p>
- </div>
- {parseProgress === 100 ? (
- <Button type="outline" onClick={removeUploadedFile}>
- Remove File
- </Button>
- ) : (
- <div className="flex w-3/5 items-center space-x-2">
- <Loader className="h-4 w-4 animate-spin" />
- <SparkBar
- value={parseProgress}
- max={100}
- type="horizontal"
- barClass="bg-green-900"
- labelBottom="Checking file..."
- labelTop={`${parseProgress}%`}
- />
- </div>
- )}
+ <div className="flex w-3/5 items-center space-x-2">
+ <Loader className="h-4 w-4 animate-spin" />
+ <SparkBar
+ value={parseProgress}
+ max={100}
+ type="horizontal"
+ barClass="bg-green-900"
+ labelBottom="Checking file..."
+ labelTop={`${parseProgress}%`}
+ />
</div>
)}
- <input ref={uploadButtonRef} className="hidden" type="file" onChange={onFileUpload} />
</div>
)
}
-
-export default SpreadSheetFileUpload
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadSheetImportConfiguration.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadSheetImportConfiguration.tsx
index 8411b06ef0b..37889c082c2 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadSheetImportConfiguration.tsx
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadSheetImportConfiguration.tsx
@@ -1,19 +1,23 @@
import { ChevronDown } from 'lucide-react'
import { useState } from 'react'
-import { Button, cn, Collapsible, SidePanel } from 'ui'
+import { Button, Checkbox_Shadcn_, cn, Collapsible, Label_Shadcn_, SidePanel } from 'ui'
import type { SpreadsheetData } from './SpreadsheetImport.types'
interface SpreadSheetImportConfigurationProps {
spreadsheetData: SpreadsheetData
selectedHeaders: string[]
onToggleHeader: (header: string) => void
+ treatEmptyAsNull: boolean
+ onToggleTreatEmptyAsNull: () => void
}
const SpreadsheetImportConfiguration = ({
spreadsheetData,
selectedHeaders,
onToggleHeader,
+ treatEmptyAsNull,
+ onToggleTreatEmptyAsNull,
}: SpreadSheetImportConfigurationProps) => {
const [expandConfiguration, setExpandConfiguration] = useState(false)
@@ -47,6 +51,19 @@ const SpreadsheetImportConfiguration = ({
By default, all columns are selected to be imported from your CSV
</p>
</div>
+ <div className="flex items-center gap-2">
+ <Checkbox_Shadcn_
+ id="treat-empty-as-null"
+ checked={treatEmptyAsNull}
+ onCheckedChange={onToggleTreatEmptyAsNull}
+ />
+ <Label_Shadcn_
+ htmlFor="treat-empty-as-null"
+ className="text-sm text-foreground-light cursor-pointer"
+ >
+ Treat empty cells as <code className="text-xs">NULL</code>
+ </Label_Shadcn_>
+ </div>
<div className="flex items-center flex-wrap gap-2 pl-0.5 pb-0.5">
{spreadsheetData.headers.map((header) => {
const isSelected = selectedHeaders.includes(header)
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.tsx
index d654897f015..ebdd9e6567c 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.tsx
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.tsx
@@ -1,32 +1,29 @@
import type { PostgresTable } from '@supabase/postgres-meta'
-import { debounce, noop } from 'lodash'
-import { useCallback, useEffect, useState } from 'react'
-import { toast } from 'sonner'
-
import { useParams } from 'common'
-import { useSendEventMutation } from 'data/telemetry/send-event-mutation'
-import { useChanged } from 'hooks/misc/useChanged'
-import { useSelectedOrganizationQuery } from 'hooks/misc/useSelectedOrganization'
-import { useTableEditorStateSnapshot } from 'state/table-editor'
+import { noop } from 'lodash'
+import { useCallback, type ChangeEvent, type DragEvent } from 'react'
+import { toast } from 'sonner'
import { SidePanel, Tabs } from 'ui'
+
import { ActionBar } from '../ActionBar'
import type { ImportContent } from '../TableEditor/TableEditor.types'
-import SpreadSheetFileUpload from './SpreadSheetFileUpload'
+import { SpreadSheetFileUpload } from './SpreadSheetFileUpload'
import SpreadsheetImportConfiguration from './SpreadSheetImportConfiguration'
+import { SpreadsheetImportPreview } from './SpreadsheetImportPreview'
import SpreadSheetTextInput from './SpreadSheetTextInput'
-import { EMPTY_SPREADSHEET_DATA } from './SpreadsheetImport.constants'
-import type { SpreadsheetData } from './SpreadsheetImport.types'
import {
- flagInvalidFileImport,
- parseSpreadsheet,
- parseSpreadsheetText,
-} from './SpreadsheetImport.utils'
-import { SpreadsheetImportPreview } from './SpreadsheetImportPreview'
+ hasAttachedFile,
+ hasAttachedText,
+ isEmptyState,
+ isFileTab,
+ isParsedState,
+ isParsingState,
+ useSpreadsheetImport,
+} from './useSpreadsheetImport'
+import { useSendEventMutation } from '@/data/telemetry/send-event-mutation'
+import { useSelectedOrganizationQuery } from '@/hooks/misc/useSelectedOrganization'
interface SpreadsheetImportProps {
- debounceDuration?: number
- headers?: string[]
- rows?: any[]
visible: boolean
selectedTable?: PostgresTable
saveContent: (prefillData: ImportContent) => void
@@ -34,14 +31,8 @@ interface SpreadsheetImportProps {
updateEditorDirty?: (value: boolean) => void
}
-const csvParseErrorMessage =
- 'Some issues have been detected. More details below the content preview.'
-
export const SpreadsheetImport = ({
visible = false,
- debounceDuration = 250,
- headers = [],
- rows = [],
selectedTable,
saveContent,
closePanel,
@@ -49,114 +40,37 @@ export const SpreadsheetImport = ({
}: SpreadsheetImportProps) => {
const { ref: projectRef } = useParams()
const { data: org } = useSelectedOrganizationQuery()
- const tableEditorSnap = useTableEditorStateSnapshot()
-
- const fileFromState =
- tableEditorSnap.sidePanel?.type === 'csv-import' ? tableEditorSnap.sidePanel.file : undefined
- const visiblityChanged = useChanged(visible)
- const [tab, setTab] = useState<'fileUpload' | 'pasteText'>('fileUpload')
- const [input, setInput] = useState<string>('')
- const [uploadedFile, setUploadedFile] = useState<File>()
- const [parseProgress, setParseProgress] = useState<number>(0)
- const [spreadsheetData, setSpreadsheetData] = useState<SpreadsheetData>({
- headers: headers,
- rows: rows,
- rowCount: 0,
- columnTypeMap: {},
- })
- const [errors, setErrors] = useState<any>([])
- const [selectedHeaders, setSelectedHeaders] = useState<string[]>([])
-
- const { mutate: sendEvent } = useSendEventMutation()
+ const {
+ state,
+ treatEmptyAsNull,
+ handleSwitchTab,
+ handleToggleTreatEmptyAsNull,
+ handleInputText,
+ handleUploadFile,
+ handleRemoveFile,
+ handleToggleSelectedHeader,
+ } = useSpreadsheetImport({ markDirty: updateEditorDirty })
+ const tab = isFileTab(state) ? 'fileUpload' : 'pasteText'
const selectedTableColumns = (selectedTable?.columns ?? []).map((column) => column.name)
- const incompatibleHeaders = selectedHeaders.filter(
- (header) => !selectedTableColumns.includes(header)
- )
- const isCompatible = selectedTable !== undefined ? incompatibleHeaders.length === 0 : true
-
- const onProgressUpdate = (progress: number) => {
- setParseProgress(progress)
- }
-
- // Process a file into table rows and columns (used for both upload and drop)
- const processFile = useCallback(
- async (file: File) => {
- updateEditorDirty(true)
- setUploadedFile(file)
- setParseProgress(0)
-
- const { headers, rowCount, columnTypeMap, errors, previewRows } = await parseSpreadsheet(
- file,
- onProgressUpdate
- )
-
- if (errors.length > 0) {
- toast.error(csvParseErrorMessage)
- }
-
- setErrors(errors)
- setSelectedHeaders(headers)
- setSpreadsheetData({ headers, rows: previewRows, rowCount, columnTypeMap })
- },
- [updateEditorDirty]
- )
-
- // Handle file upload events from file input
- const onFileUpload = useCallback(
- async (event: any) => {
- event.persist()
- const [file] = event.target.files || event.dataTransfer.files
- if (file && !flagInvalidFileImport(file)) {
- await processFile(file)
- }
- event.target.value = ''
- },
- [processFile]
- )
-
- const resetSpreadsheetImport = useCallback(() => {
- setInput('')
- setSpreadsheetData(EMPTY_SPREADSHEET_DATA)
- setUploadedFile(undefined)
- setErrors([])
- updateEditorDirty(false)
- }, [updateEditorDirty])
-
- const readSpreadsheetText = async (text: string) => {
- if (text.length > 0) {
- const { headers, rows, columnTypeMap, errors } = await parseSpreadsheetText(text)
- if (errors.length > 0) {
- toast.error(csvParseErrorMessage)
- }
- setErrors(errors)
- setSelectedHeaders(headers)
- setSpreadsheetData({ headers, rows, rowCount: rows.length, columnTypeMap })
- } else {
- setSpreadsheetData(EMPTY_SPREADSHEET_DATA)
- }
- }
-
- // eslint-disable-next-line react-hooks/exhaustive-deps
- const handler = useCallback(debounce(readSpreadsheetText, debounceDuration), [])
- const onInputChange = (event: any) => {
- setInput(event.target.value)
- handler(event.target.value)
- }
-
- const onToggleHeader = (header: string) => {
- const updatedSelectedHeaders = selectedHeaders.includes(header)
- ? selectedHeaders.filter((h) => h !== header)
- : selectedHeaders.concat([header])
- setSelectedHeaders(updatedSelectedHeaders)
- }
+ const csvHeaders =
+ state._tag === 'file_parsed' || state._tag === 'text_parsed' ? state.data.headers : []
+ const incompatibleHeaders = csvHeaders.filter((header) => !selectedTableColumns.includes(header))
+ const isCompatible = !selectedTable || incompatibleHeaders.length === 0
+ const { mutate: sendEvent } = useSendEventMutation()
const onConfirm = (resolve: () => void) => {
- if (tab === 'fileUpload' && uploadedFile === undefined) {
+ if (state._tag === 'no_selected_file') {
toast.error('Please upload a file to import your data with')
resolve()
- } else if (selectedHeaders.length === 0) {
+ } else if (state._tag === 'no_pasted_text') {
+ toast.error('Please paste some CSV text to import')
+ resolve()
+ } else if (state._tag === 'parsing_file' || state._tag === 'parsing_text') {
+ toast.error('Your data is still being processed, please wait a moment')
+ resolve()
+ } else if (state.selectedHeaders.length === 0) {
toast.error('Please select at least one header from your CSV')
resolve()
} else if (!isCompatible) {
@@ -165,7 +79,13 @@ export const SpreadsheetImport = ({
)
resolve()
} else {
- saveContent({ file: uploadedFile, ...spreadsheetData, selectedHeaders, resolve })
+ saveContent({
+ file: state._tag === 'file_parsed' ? state.file : undefined,
+ ...state.data,
+ selectedHeaders: state.selectedHeaders,
+ treatEmptyAsNull: treatEmptyAsNull,
+ resolve,
+ })
sendEvent({
action: 'import_data_added',
groups: { project: projectRef ?? 'Unknown', organization: org?.slug ?? 'Unknown' },
@@ -173,20 +93,6 @@ export const SpreadsheetImport = ({
}
}
- useEffect(() => {
- if (visiblityChanged && visible) {
- if (fileFromState) processFile(fileFromState)
- else if (headers.length === 0) resetSpreadsheetImport()
- }
- }, [
- visiblityChanged,
- visible,
- fileFromState,
- processFile,
- headers.length,
- resetSpreadsheetImport,
- ])
-
return (
<SidePanel
size="large"
@@ -213,37 +119,44 @@ export const SpreadsheetImport = ({
>
<SidePanel.Content>
<div className="pt-6">
- <Tabs block type="pills" onChange={setTab}>
+ <Tabs block type="pills" activeId={tab} onChange={handleSwitchTab}>
<Tabs.Panel id="fileUpload" label="Upload CSV">
- <SpreadSheetFileUpload
- parseProgress={parseProgress}
- uploadedFile={uploadedFile}
- onFileUpload={onFileUpload}
- removeUploadedFile={resetSpreadsheetImport}
+ <SpreadsheetFileDropZone
+ file={hasAttachedFile(state) ? state.file : undefined}
+ parseProgress={
+ isEmptyState(state) ? 0 : isParsingState(state) ? state.progress : 100
+ }
+ onUploadFile={handleUploadFile}
+ onRemoveFile={handleRemoveFile}
/>
</Tabs.Panel>
<Tabs.Panel id="pasteText" label="Paste text">
- <SpreadSheetTextInput input={input} onInputChange={onInputChange} />
+ <SpreadsheetInputZone
+ text={hasAttachedText(state) ? state.text : ''}
+ onInputChange={handleInputText}
+ />
</Tabs.Panel>
</Tabs>
</div>
</SidePanel.Content>
- {spreadsheetData.headers.length > 0 && (
+ {isParsedState(state) && state.data.headers.length > 0 && (
<>
<div className="pt-4">
<SidePanel.Separator />
</div>
<SpreadsheetImportConfiguration
- spreadsheetData={spreadsheetData}
- selectedHeaders={selectedHeaders}
- onToggleHeader={onToggleHeader}
+ spreadsheetData={state.data}
+ selectedHeaders={state.selectedHeaders}
+ onToggleHeader={handleToggleSelectedHeader}
+ treatEmptyAsNull={treatEmptyAsNull}
+ onToggleTreatEmptyAsNull={handleToggleTreatEmptyAsNull}
/>
<SidePanel.Separator />
<SpreadsheetImportPreview
selectedTable={selectedTable}
- spreadsheetData={spreadsheetData}
- errors={errors}
- selectedHeaders={selectedHeaders}
+ spreadsheetData={state.data}
+ errors={state.errors}
+ selectedHeaders={state.selectedHeaders}
incompatibleHeaders={incompatibleHeaders}
/>
<SidePanel.Separator />
@@ -252,3 +165,59 @@ export const SpreadsheetImport = ({
</SidePanel>
)
}
+
+function isDragEvent(
+ event: DragEvent<HTMLDivElement> | ChangeEvent<HTMLInputElement>
+): event is DragEvent<HTMLDivElement> {
+ return event.type === 'drop'
+}
+
+interface SpreadsheetFileDropZoneProps {
+ file: File | undefined
+ parseProgress: number
+ onUploadFile: (file: File) => void
+ onRemoveFile: () => void
+}
+
+function SpreadsheetFileDropZone({
+ file,
+ parseProgress,
+ onUploadFile,
+ onRemoveFile,
+}: SpreadsheetFileDropZoneProps) {
+ const handleUploadFile = useCallback(
+ async function uploadFile(event: DragEvent<HTMLDivElement> | ChangeEvent<HTMLInputElement>) {
+ const [file] = isDragEvent(event) ? event.dataTransfer.files : (event.target.files ?? [])
+ if (!file) {
+ return
+ }
+ onUploadFile(file)
+ },
+ [onUploadFile]
+ )
+
+ return (
+ <SpreadSheetFileUpload
+ uploadedFile={file}
+ onFileUpload={handleUploadFile}
+ parseProgress={parseProgress}
+ removeUploadedFile={onRemoveFile}
+ />
+ )
+}
+
+interface SpreadsheetInputZoneProps {
+ text: string
+ onInputChange: (text: string) => void
+}
+
+function SpreadsheetInputZone({ text, onInputChange }: SpreadsheetInputZoneProps) {
+ const handleInputText = useCallback(
+ function inputText(event: ChangeEvent<HTMLTextAreaElement>) {
+ onInputChange(event.target.value)
+ },
+ [onInputChange]
+ )
+
+ return <SpreadSheetTextInput input={text} onInputChange={handleInputText} />
+}
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.types.ts b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.types.ts
index 54771df8f74..54bd0a4702c 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.types.ts
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.types.ts
@@ -1,8 +1,10 @@
import type { Dictionary } from 'types'
+import type { InferredColumnType } from './SpreadsheetImport.utils'
+
export interface SpreadsheetData {
headers: string[]
- rows: any[]
+ rows: unknown[]
rowCount: number
- columnTypeMap: Dictionary<string>
+ columnTypeMap: Dictionary<InferredColumnType>
}
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.utils.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.utils.tsx
index d601a0b490c..ad97b140828 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.utils.tsx
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.utils.tsx
@@ -1,36 +1,50 @@
import dayjs from 'dayjs'
-import { has, includes } from 'lodash'
+import { DOCS_URL } from 'lib/constants'
+import { isObject, tryParseJson } from 'lib/helpers'
+import { has } from 'lodash'
import { ExternalLink } from 'lucide-react'
import Link from 'next/link'
import Papa from 'papaparse'
import { toast } from 'sonner'
-
-import { DOCS_URL } from 'lib/constants'
-import { tryParseJson } from 'lib/helpers'
import { Button } from 'ui'
+
import {
MAX_TABLE_EDITOR_IMPORT_CSV_SIZE,
UPLOAD_FILE_EXTENSIONS,
UPLOAD_FILE_TYPES,
} from './SpreadsheetImport.constants'
+import type { SpreadsheetData } from './SpreadsheetImport.types'
const CHUNK_SIZE = 1024 * 1024 * 0.25 // 0.25MB
-export const parseSpreadsheetText: any = (text: string) => {
- const columnTypeMap: Record<any, any> = {}
- let previewRows: any[] = []
+export function parseSpreadsheetText({
+ text,
+ treatEmptyAsNull = false,
+}: {
+ text: string
+ treatEmptyAsNull?: boolean
+}): Promise<{
+ headers: Array<string>
+ rows: Array<unknown>
+ previewRows: Array<unknown>
+ columnTypeMap: Record<string, InferredColumnType>
+ errors: Array<Papa.ParseError & { data: unknown }>
+}> {
+ const columnTypeMap: Record<string, InferredColumnType> = {}
+ let previewRows: unknown[] = []
return new Promise((resolve) => {
Papa.parse(text, {
header: true,
dynamicTyping: false,
skipEmptyLines: true,
+ transform: treatEmptyAsNull ? (value: string) => (value === '' ? null : value) : undefined,
complete: (results) => {
const headers = results.meta.fields || []
const rows = results.data
- const errors = results.errors
+ const errors = results.errors.map((error) => ({ ...error, data: results.data[error.row] }))
headers.forEach((header) => {
- const type = inferColumnType(header, results.data as any[])
+ const type = inferColumnType(header, results.data)
if (!has(columnTypeMap, header)) {
columnTypeMap[header] = type
} else if (columnTypeMap[header] !== type) {
@@ -52,15 +66,21 @@ export const parseSpreadsheetText: any = (text: string) => {
*/
export const parseSpreadsheet = (
file: File,
- onProgressUpdate: (progress: number) => void
-): Promise<any> => {
+ onProgressUpdate: (progress: number) => void,
+ treatEmptyAsNull = false
+): Promise<
+ Omit<SpreadsheetData, 'rows'> & {
+ previewRows: Array<unknown>
+ errors: Array<Papa.ParseError & { data: unknown }>
+ }
+> => {
let headers: string[] = []
let chunkNumber = 0
let rowCount = 0
- let previewRows: any[] = []
+ let previewRows: unknown[] = []
- const columnTypeMap: Record<any, any> = {}
- const errors: any[] = []
+ const columnTypeMap: Record<string, InferredColumnType> = {}
+ const errors: (Papa.ParseError & { data: unknown })[] = []
return new Promise((resolve) => {
Papa.parse(file, {
@@ -71,10 +91,21 @@ export const parseSpreadsheet = (
quoteChar: file.type === 'text/tab-separated-values' ? '' : '"',
chunkSize: CHUNK_SIZE,
chunk: (results) => {
- headers = results.meta.fields as string[]
+ headers = results.meta.fields ?? []
+
+ // transform option is silently ignored when worker: true, so we apply
+ // the empty → null conversion manually here instead
+ const data = treatEmptyAsNull
+ ? results.data.map((row) => {
+ const rowAsObject = isObject(row) ? row : {}
+ return Object.fromEntries(
+ Object.entries(rowAsObject).map(([k, v]) => [k, v === '' ? null : v])
+ )
+ })
+ : results.data
headers.forEach((header) => {
- const type = inferColumnType(header, results.data as any[])
+ const type = inferColumnType(header, data)
if (!has(columnTypeMap, header)) {
columnTypeMap[header] = type
} else if (columnTypeMap[header] !== type) {
@@ -82,12 +113,16 @@ export const parseSpreadsheet = (
}
})
- rowCount += results.data.length
- previewRows = results.data.slice(0, 20)
+ if (chunkNumber === 0) {
+ previewRows = data.slice(0, 20)
+ }
+ rowCount += data.length
if (results.errors.length > 0) {
- const formattedErrors = results.errors.map((error) => {
- if (error) return { ...error, data: results.data[error.row] }
- })
+ const formattedErrors = results.errors
+ .map((error) => {
+ if (error) return { ...error, data: results.data[error.row] }
+ })
+ .filter((error) => error !== undefined)
errors.push(...formattedErrors)
}
@@ -107,7 +142,9 @@ export const revertSpreadsheet = (headers: string[], rows: any[]) => {
return Papa.unparse(rows, { columns: headers })
}
-export const inferColumnType = (column: string, rows: object[]) => {
+export type InferredColumnType = 'int8' | 'float8' | 'bool' | 'jsonb' | 'timestamptz' | 'text'
+
+export const inferColumnType = (column: string, rows: unknown[]): InferredColumnType => {
// General strategy is to check the first row first, before checking across all the rows
// to ensure uniformity in data type. Thinking we do this as an optimization instead of
// checking all the rows up front.
@@ -115,8 +152,13 @@ export const inferColumnType = (column: string, rows: object[]) => {
// If there are no rows to infer for, default to text
if (rows.length === 0) return 'text'
- const columnData = (rows[0] as any)[column]
- const columnDataAcrossRows = rows.map((row: object) => (row as any)[column])
+ const firstRow = rows[0]
+ if (!isObject(firstRow)) {
+ return 'text'
+ }
+
+ const columnData = firstRow[column]
+ const columnDataAcrossRows = rows.map((row) => (isObject(row) ? row[column] : undefined))
// Unable to infer any type as there's no data, default to text
if (columnData === undefined || columnData === null) {
@@ -124,8 +166,9 @@ export const inferColumnType = (column: string, rows: object[]) => {
}
// Infer numerical data type (defaults to either int8 or float8)
- if (Number(columnData)) {
- const columnNumberCheck = rows.map((row: object) => Number((row as any)[column]))
+ const columnAsNumber = Number(columnData)
+ if (!Number.isNaN(columnAsNumber)) {
+ const columnNumberCheck = rows.map((row) => (isObject(row) ? Number(row[column]) : NaN))
if (columnNumberCheck.includes(NaN)) {
return 'text'
} else {
@@ -135,10 +178,14 @@ export const inferColumnType = (column: string, rows: object[]) => {
}
// Infer boolean type
- if (includes(['true', 'false'], columnData.toString().toLowerCase())) {
- const isAllBoolean = columnDataAcrossRows.every((item: any) => {
- if (item === null || item === undefined) return true
- else return includes(['true', 'false'], item.toString().toLowerCase())
+ const firstCellLowerCase = columnData.toString().toLowerCase()
+ if (firstCellLowerCase === 'true' || firstCellLowerCase === 'false') {
+ const isAllBoolean = columnDataAcrossRows.every((item) => {
+ if (item === null || item === undefined) {
+ return true
+ }
+ const cellLowerCase = item.toString().toLowerCase()
+ return cellLowerCase === 'true' || cellLowerCase === 'false'
})
if (isAllBoolean) {
return 'bool'
@@ -147,16 +194,16 @@ export const inferColumnType = (column: string, rows: object[]) => {
// Infer json type
if (tryParseJson(columnData)) {
- const isAllJson = columnDataAcrossRows.every((item: any) => tryParseJson(columnData))
+ const isAllJson = columnDataAcrossRows.every((item) => tryParseJson(item))
if (isAllJson) {
return 'jsonb'
}
}
// Infer datetime type
- if (Date.parse(columnData)) {
+ if (Date.parse(String(columnData))) {
const isAllTimestamptz = columnDataAcrossRows.every((item) =>
- dayjs(item, 'YYYY-MM-DD hh:mm:ss').isValid()
+ dayjs(String(item), 'YYYY-MM-DD hh:mm:ss').isValid()
)
if (isAllTimestamptz) {
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetPreviewGrid.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetPreviewGrid.tsx
index e8376a9e36e..f003d23a5d6 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetPreviewGrid.tsx
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetPreviewGrid.tsx
@@ -39,7 +39,7 @@ const SpreadsheetPreviewGrid = ({
</div>
),
renderCell: ({ row }: { row: any }) => {
- const isEmpty = !row[header]
+ const isEmpty = row[header] === null || row[header] === undefined
return (
<span className={cn('text-sm flex items-center', isEmpty && 'text-foreground-light')}>
{isEmpty ? 'NULL' : row[header]}
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/useSpreadsheetImport.ts b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/useSpreadsheetImport.ts
new file mode 100644
index 00000000000..abad9790e75
--- /dev/null
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/useSpreadsheetImport.ts
@@ -0,0 +1,302 @@
+import { debounce, noop } from 'lodash'
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
+import { toast } from 'sonner'
+
+import type { SpreadsheetData } from './SpreadsheetImport.types'
+import { parseSpreadsheet, parseSpreadsheetText } from './SpreadsheetImport.utils'
+import { useStaticEffectEvent } from '@/hooks/useStaticEffectEvent'
+import { useTableEditorStateSnapshot } from '@/state/table-editor'
+
+interface NoFileState {
+ _tag: 'no_selected_file'
+}
+
+interface NoInputState {
+ _tag: 'no_pasted_text'
+}
+
+interface ParsingFileState {
+ _tag: 'parsing_file'
+ file: File
+ progress: number
+}
+
+interface ParsingTextState {
+ _tag: 'parsing_text'
+ text: string
+ progress: number
+}
+
+interface FileParsedState {
+ _tag: 'file_parsed'
+ file: File
+ selectedHeaders: Array<string>
+ data: SpreadsheetData
+ errors: Array<Papa.ParseError & { data: unknown }>
+}
+
+interface TextParsedState {
+ _tag: 'text_parsed'
+ text: string
+ selectedHeaders: Array<string>
+ data: SpreadsheetData
+ errors: Array<Papa.ParseError & { data: unknown }>
+}
+
+type FileUploadState =
+ | NoFileState
+ | ParsingFileState
+ | FileParsedState
+ | NoInputState
+ | ParsingTextState
+ | TextParsedState
+
+export function isFileTab(
+ state: FileUploadState
+): state is NoFileState | ParsingFileState | FileParsedState {
+ return (
+ state._tag === 'no_selected_file' ||
+ state._tag === 'parsing_file' ||
+ state._tag === 'file_parsed'
+ )
+}
+
+export function hasAttachedFile(
+ state: FileUploadState
+): state is ParsingFileState | FileParsedState {
+ return state._tag === 'parsing_file' || state._tag === 'file_parsed'
+}
+
+export function hasAttachedText(
+ state: FileUploadState
+): state is ParsingTextState | TextParsedState {
+ return state._tag === 'parsing_text' || state._tag === 'text_parsed'
+}
+
+export function isEmptyState(state: FileUploadState): state is NoFileState | NoInputState {
+ return state._tag === 'no_selected_file' || state._tag === 'no_pasted_text'
+}
+
+export function isParsingState(
+ state: FileUploadState
+): state is ParsingFileState | ParsingTextState {
+ return state._tag === 'parsing_file' || state._tag === 'parsing_text'
+}
+
+export function isParsedState(state: FileUploadState): state is FileParsedState | TextParsedState {
+ return state._tag === 'file_parsed' || state._tag === 'text_parsed'
+}
+
+const csvParseErrorMessage =
+ 'Some issues have been detected. More details below the content preview.'
+
+interface UseSpreadsheetImportParameters {
+ markDirty?: (dirty: boolean) => void
+ debounceDuration?: number
+}
+
+export function useSpreadsheetImport({
+ markDirty = noop,
+ debounceDuration = 250,
+}: UseSpreadsheetImportParameters) {
+ const abortControllerRef = useRef<AbortController>(new AbortController())
+ const resetAbortController = useCallback(function resetAbortController() {
+ abortControllerRef.current.abort()
+ abortControllerRef.current = new AbortController()
+ return abortControllerRef.current.signal
+ }, [])
+
+ const tableEditorSnap = useTableEditorStateSnapshot()
+ const fileFromGlobalState =
+ tableEditorSnap.sidePanel?.type === 'csv-import' ? tableEditorSnap.sidePanel.file : undefined
+
+ const [historicalState, setHistoricalState] = useState<FileUploadState>()
+ const [state, setState] = useState<FileUploadState>(function initializeState() {
+ if (fileFromGlobalState) {
+ return {
+ _tag: 'parsing_file',
+ file: fileFromGlobalState,
+ progress: 0,
+ }
+ } else {
+ return { _tag: 'no_selected_file' }
+ }
+ })
+
+ const [treatEmptyAsNull, setTreatEmptyAsNull] = useState(false)
+
+ const handleSwitchTab = useCallback(
+ function switchTab() {
+ if (isFileTab(state)) {
+ setState(historicalState ?? { _tag: 'no_pasted_text' })
+ } else {
+ setState(historicalState ?? { _tag: 'no_selected_file' })
+ }
+ setHistoricalState(state)
+ },
+ [state, historicalState]
+ )
+
+ const setParseProgress = useCallback(function setParseProgress(progress: number) {
+ setState((prev) => {
+ if (prev._tag === 'parsing_file' || prev._tag === 'parsing_text') {
+ return { ...prev, progress }
+ }
+ return prev
+ })
+ }, [])
+
+ const processFile = useCallback(
+ async function processFile(file: File, { treatEmptyAsNull }: { treatEmptyAsNull: boolean }) {
+ const currentSignal = resetAbortController()
+
+ const { headers, rowCount, columnTypeMap, errors, previewRows } = await parseSpreadsheet(
+ file,
+ setParseProgress,
+ treatEmptyAsNull
+ )
+ if (currentSignal.aborted) return
+
+ if (errors.length > 0) {
+ toast.error(csvParseErrorMessage)
+ }
+
+ setState({
+ _tag: 'file_parsed',
+ file,
+ selectedHeaders: headers,
+ data: { headers, rows: previewRows, rowCount, columnTypeMap },
+ errors,
+ })
+ },
+ [resetAbortController, setParseProgress]
+ )
+
+ const processText = useCallback(
+ async function processText(text: string, { treatEmptyAsNull }: { treatEmptyAsNull: boolean }) {
+ const currentSignal = resetAbortController()
+ const { headers, rows, columnTypeMap, errors } = await parseSpreadsheetText({
+ text,
+ treatEmptyAsNull,
+ })
+ if (currentSignal.aborted) return
+
+ if (errors.length > 0) {
+ toast.error(csvParseErrorMessage)
+ }
+
+ setState({
+ _tag: 'text_parsed',
+ text,
+ selectedHeaders: headers,
+ data: { headers, rows, rowCount: rows.length, columnTypeMap },
+ errors,
+ })
+ },
+ [resetAbortController]
+ )
+ const processTextStable = useStaticEffectEvent(processText)
+ const processTextDebounced = useMemo(
+ () => debounce(processTextStable, debounceDuration),
+ [debounceDuration, processTextStable]
+ )
+
+ const cleanup = useStaticEffectEvent(function cleanup() {
+ processTextDebounced.cancel()
+ abortControllerRef.current.abort()
+ })
+ useEffect(() => {
+ return cleanup
+ }, [cleanup])
+
+ // When the component mounts with a file already in global state (e.g. dropped onto the
+ // grid), the useState initializer above sets _tag to 'parsing_file' but does not start
+ // the actual parse. Kick it off here. useStaticEffectEvent ensures we always read the
+ // latest state/treatEmptyAsNull without re-triggering the effect.
+ const processOnMount = useStaticEffectEvent(async function processOnMount() {
+ if (state._tag === 'parsing_file') {
+ await processFile(state.file, { treatEmptyAsNull })
+ }
+ })
+ useEffect(() => {
+ processOnMount()
+ }, [processOnMount])
+
+ const processSpreadsheet = useCallback(
+ async function processSpreadsheet({ treatEmptyAsNull }: { treatEmptyAsNull: boolean }) {
+ if (hasAttachedFile(state)) {
+ return processFile(state.file, { treatEmptyAsNull })
+ } else if (hasAttachedText(state)) {
+ return processText(state.text, { treatEmptyAsNull })
+ }
+ },
+ [state, processFile, processText]
+ )
+
+ const handleToggleTreatEmptyAsNull = useCallback(
+ function toggleTreatEmptyAsNull() {
+ const next = !treatEmptyAsNull
+ setTreatEmptyAsNull(next)
+ processSpreadsheet({ treatEmptyAsNull: next })
+ },
+ [treatEmptyAsNull, processSpreadsheet]
+ )
+
+ const handleUploadFile = useCallback(
+ async function uploadFile(file: File) {
+ markDirty(true)
+ setState({ _tag: 'parsing_file', file, progress: 0 })
+ await processFile(file, { treatEmptyAsNull })
+ },
+ [treatEmptyAsNull, markDirty, processFile]
+ )
+
+ const handleInputText = useCallback(
+ async function inputText(text: string) {
+ if (text.length === 0) {
+ resetAbortController()
+ markDirty(false)
+ return setState({ _tag: 'no_pasted_text' })
+ } else {
+ markDirty(true)
+ setState({ _tag: 'parsing_text', text, progress: 0 })
+ await processTextDebounced(text, { treatEmptyAsNull })
+ }
+ },
+ [treatEmptyAsNull, resetAbortController, markDirty, processTextDebounced]
+ )
+
+ const handleRemoveFile = useCallback(
+ function removeFile() {
+ resetAbortController()
+ markDirty(false)
+ setState({ _tag: 'no_selected_file' })
+ },
+ [resetAbortController, markDirty]
+ )
+
+ const handleToggleSelectedHeader = useCallback(
+ function toggleSelectedHeader(header: string) {
+ if (state._tag === 'file_parsed' || state._tag === 'text_parsed') {
+ if (!state.data.headers.includes(header)) return
+
+ const selectedHeaders = state.selectedHeaders.includes(header)
+ ? state.selectedHeaders.filter((h) => h !== header)
+ : [...state.selectedHeaders, header]
+ setState({ ...state, selectedHeaders })
+ }
+ },
+ [state]
+ )
+
+ return {
+ state,
+ treatEmptyAsNull,
+ handleSwitchTab,
+ handleToggleTreatEmptyAsNull,
+ handleToggleSelectedHeader,
+ handleUploadFile,
+ handleInputText,
+ handleRemoveFile,
+ }
+}
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/TableEditor/TableEditor.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/TableEditor/TableEditor.tsx
index c3908992673..0013b143e12 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/TableEditor/TableEditor.tsx
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/TableEditor/TableEditor.tsx
@@ -10,6 +10,7 @@ import { useIsFeatureEnabled } from 'hooks/misc/useIsFeatureEnabled'
import { useQuerySchemaState } from 'hooks/misc/useSchemaQueryState'
import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'
import { useUrlState } from 'hooks/ui/useUrlState'
+import { useVisibleKey } from 'hooks/ui/useVisibleKey'
import { useProtectedSchemas } from 'hooks/useProtectedSchemas'
import { DOCS_URL } from 'lib/constants'
import { useTrack } from 'lib/telemetry/track'
@@ -94,6 +95,7 @@ export const TableEditor = ({
const [isDuplicateRows, setIsDuplicateRows] = useState<boolean>(false)
const [importContent, setImportContent] = useState<ImportContent>()
const [isImportingSpreadsheet, setIsImportingSpreadsheet] = useState<boolean>(false)
+ const spreadsheetImportKey = useVisibleKey(isImportingSpreadsheet)
const [rlsConfirmVisible, setRlsConfirmVisible] = useState<boolean>(false)
const { data: types } = useEnumeratedTypesQuery({
@@ -487,9 +489,8 @@ export const TableEditor = ({
)}
<SpreadsheetImport
+ key={spreadsheetImportKey}
visible={isImportingSpreadsheet}
- headers={importContent?.headers}
- rows={importContent?.rows}
saveContent={(prefillData: ImportContent) => {
setImportContent(prefillData)
setIsImportingSpreadsheet(false)
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/TableEditor/TableEditor.types.ts b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/TableEditor/TableEditor.types.ts
index 5db5952c543..10868afdd61 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/TableEditor/TableEditor.types.ts
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/TableEditor/TableEditor.types.ts
@@ -1,6 +1,8 @@
-import type { Prettify } from '@/lib/type-helpers'
import type { Dictionary } from 'types'
+
import type { ColumnField } from '../SidePanelEditor.types'
+import type { InferredColumnType } from '../SpreadsheetImport/SpreadsheetImport.utils'
+import type { Prettify } from '@/lib/type-helpers'
export type TableField = Prettify<{
id: number
@@ -15,8 +17,9 @@ export interface ImportContent {
file?: File
headers: string[]
rowCount: number
- rows: object[]
- columnTypeMap: Dictionary<any>
+ rows: unknown[]
+ columnTypeMap: Dictionary<InferredColumnType>
selectedHeaders: string[]
+ treatEmptyAsNull: boolean
resolve: () => void
}
diff --git a/apps/studio/hooks/ui/useVisibleKey.ts b/apps/studio/hooks/ui/useVisibleKey.ts
new file mode 100644
index 00000000000..ab20eac11f3
--- /dev/null
+++ b/apps/studio/hooks/ui/useVisibleKey.ts
@@ -0,0 +1,20 @@
+import { useState } from 'react'
+
+/**
+ * Returns a key that increments each time `isVisible` transitions from false to true.
+ * Use this as the `key` prop on a component to force a clean remount on each open,
+ * instead of a `useEffect` that imperatively resets internal state.
+ */
+export function useVisibleKey(isVisible: boolean): number {
+ const [key, setKey] = useState(0)
+ const [prevIsVisible, setPrevIsVisible] = useState(false)
+
+ if (isVisible && !prevIsVisible) {
+ setPrevIsVisible(true)
+ setKey((k) => k + 1)
+ } else if (!isVisible && prevIsVisible) {
+ setPrevIsVisible(false)
+ }
+
+ return key
+}
diff --git a/apps/studio/lib/helpers.ts b/apps/studio/lib/helpers.ts
index 23d7437f36d..102a1b6a242 100644
--- a/apps/studio/lib/helpers.ts
+++ b/apps/studio/lib/helpers.ts
@@ -428,3 +428,16 @@ export const createWrappedSymbol = (name: string, display: string): Symbol => {
// unreachable.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export function neverGuard(_: never): any {}
+
+export function isObject(
+ maybeObject: unknown
+): maybeObject is Record<string | symbol | number, unknown> {
+ return maybeObject !== null && typeof maybeObject === 'object' && !Array.isArray(maybeObject)
+}
+
+export function isObjectContainingKeys<T extends string | symbol | number>(
+ maybeObject: unknown,
+ keys: Array<T>
+): maybeObject is { [K in T]: unknown } {
+ return isObject(maybeObject) && keys.every((key) => key in maybeObject)
+}
diff --git a/apps/studio/tests/components/Editor/SpreadsheetImport.utils.test.ts b/apps/studio/tests/components/Editor/SpreadsheetImport.utils.test.ts
index 0660e7dfc82..4eeeacc3791 100644
--- a/apps/studio/tests/components/Editor/SpreadsheetImport.utils.test.ts
+++ b/apps/studio/tests/components/Editor/SpreadsheetImport.utils.test.ts
@@ -1,9 +1,12 @@
-import { inferColumnType } from 'components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.utils'
-import { describe, test, expect } from 'vitest'
+import {
+ inferColumnType,
+ parseSpreadsheetText,
+} from 'components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.utils'
+import { describe, expect, test } from 'vitest'
-describe('SpreadsheedImport.utils: inferColumnType', () => {
+describe('SpreadsheetImport.utils: inferColumnType', () => {
test('should default column type to text if no rows to infer from', () => {
- const mockData: any[] = []
+ const mockData: Array<unknown> = []
const type = inferColumnType('id', mockData)
expect(type).toBe('text')
})
@@ -65,3 +68,45 @@ describe('SpreadsheedImport.utils: inferColumnType', () => {
expect(type4).toBe('timestamptz')
})
})
+
+interface SampleRow {
+ name: string
+ age: string | null
+ city?: string | null
+}
+
+describe('SpreadsheetImport.utils: parseSpreadsheetText', () => {
+ test('should keep empty cells as empty strings by default', async () => {
+ const csv = `name,age\nJohn,25\nJane,`
+ const { rows } = await parseSpreadsheetText({ text: csv })
+ expect((rows[1] as SampleRow).age).toBe('')
+ })
+
+ test('should convert empty cells to null when treatEmptyAsNull is true', async () => {
+ const csv = `name,age\nJohn,25\nJane,`
+ const { rows } = await parseSpreadsheetText({ text: csv, treatEmptyAsNull: true })
+ expect((rows[1] as SampleRow).age).toBeNull()
+ })
+
+ test('should not affect non-empty values when treatEmptyAsNull is true', async () => {
+ const csv = `name,age\nJohn,25\nJane,`
+ const { rows } = await parseSpreadsheetText({ text: csv, treatEmptyAsNull: true })
+ expect((rows[0] as SampleRow).name).toBe('John')
+ expect((rows[0] as SampleRow).age).toBe('25')
+ })
+
+ test('should handle multiple empty cells across columns when treatEmptyAsNull is true', async () => {
+ const csv = `name,age,city\nJohn,,\nJane,30,`
+ const { rows } = await parseSpreadsheetText({ text: csv, treatEmptyAsNull: true })
+ expect((rows[0] as SampleRow).age).toBeNull()
+ expect((rows[0] as SampleRow).city).toBeNull()
+ expect((rows[1] as SampleRow).age).toBe('30')
+ expect((rows[1] as SampleRow).city).toBeNull()
+ })
+
+ test('should return correct headers regardless of treatEmptyAsNull', async () => {
+ const csv = `name,age\nJohn,25`
+ const { headers } = await parseSpreadsheetText({ text: csv, treatEmptyAsNull: true })
+ expect(headers).toEqual(['name', 'age'])
+ })
+})