Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

chore: improve storage migration script (#33927)

Illia Basalaiev9b1377c2ba71
diff --git a/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdx b/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdx
index a012aaa51aa..c4c2275fcb9 100644
--- a/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdx
+++ b/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdx
@@ -236,47 +236,325 @@ const OLD_PROJECT_SERVICE_KEY = 'old-project-service-key-xxx'
const NEW_PROJECT_URL = 'https://yyy.supabase.co'
const NEW_PROJECT_SERVICE_KEY = 'new-project-service-key-yyy'
-;(async () => {
- const oldSupabaseRestClient = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY, {
- db: {
- schema: 'storage',
+const oldSupabase = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY)
+const newSupabase = createClient(NEW_PROJECT_URL, NEW_PROJECT_SERVICE_KEY)
+
+function createLoadingAnimation(message) {
+ const readline = require('readline')
+ const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
+ let i = 0
+ let timer
+ let stopped = false
+
+ const animate = () => {
+ if (stopped) return
+ process.stdout.write(`\r${frames[i]} ${message}`)
+ i = (i + 1) % frames.length
+ timer = setTimeout(animate, 80)
+ }
+
+ animate()
+
+ return {
+ stop: (finalMessage = '') => {
+ stopped = true
+ clearTimeout(timer)
+ readline.clearLine(process.stdout, 0)
+ readline.cursorTo(process.stdout, 0)
+ process.stdout.write(`✓ ${finalMessage || message}\n`)
},
- })
- const oldSupabaseClient = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY)
- const newSupabaseClient = createClient(NEW_PROJECT_URL, NEW_PROJECT_SERVICE_KEY)
-
- // make sure you update max_rows in postgrest settings if you have a lot of objects
- // or paginate here
- const { data: oldObjects, error } = await oldSupabaseRestClient.from('objects').select()
- if (error) {
- console.log('error getting objects from old bucket')
- throw error
}
+}
+
+/**
+ * Lists all files in a bucket, handling nested folders recursively.
+ */
+async function listAllFiles(bucket, path = '') {
+ const loader = createLoadingAnimation(`Listing files in '${bucket}${path ? '/' + path : ''}'...`)
+
+ try {
+ const { data, error } = await oldSupabase.storage.from(bucket).list(path, { limit: 1000 })
+ if (error) {
+ loader.stop(`Error listing files in '${bucket}${path ? '/' + path : ''}'`)
+ throw new Error(`❌ Error listing files in bucket '${bucket}': ${error.message}`)
+ }
+
+ if (!data || data.length === 0) {
+ loader.stop(`No files found in '${bucket}${path ? '/' + path : ''}'`)
+ return []
+ }
- for (const objectData of oldObjects) {
- console.log(`moving ${objectData.id}`)
- try {
- const { data, error: downloadObjectError } = await oldSupabaseClient.storage
- .from(objectData.bucket_id)
- .download(objectData.name)
- if (downloadObjectError) {
- throw downloadObjectError
+ let files = []
+ for (const item of data) {
+ if (!item.metadata) {
+ loader.stop(`Found folder '${item.name}' in '${bucket}${path ? '/' + path : ''}'`)
+ const subFiles = await listAllFiles(bucket, `${path}${item.name}/`)
+ files = files.concat(subFiles)
+ } else {
+ files.push({ fullPath: `${path}${item.name}`, metadata: item.metadata })
}
+ }
+
+ loader.stop(`Found ${files.length} files in '${bucket}${path ? '/' + path : ''}'`)
+ return files
+ } catch (error) {
+ loader.stop()
+ throw error
+ }
+}
+
+/**
+ * Creates a bucket in the new Supabase project if it doesn't exist.
+ */
+async function ensureBucketExists(bucketName, options = {}) {
+ const { data: existingBucket, error: getBucketError } =
+ await newSupabase.storage.getBucket(bucketName)
+
+ if (getBucketError && !getBucketError.message.includes('not found')) {
+ throw new Error(`❌ Error checking if bucket '${bucketName}' exists: ${getBucketError.message}`)
+ }
- const { _, error: uploadObjectError } = await newSupabaseClient.storage
- .from(objectData.bucket_id)
- .upload(objectData.name, data, {
- upsert: true,
- contentType: objectData.metadata.mimetype,
- cacheControl: objectData.metadata.cacheControl,
- })
- if (uploadObjectError) {
- throw uploadObjectError
+ if (!existingBucket) {
+ console.log(`🪣 Creating bucket '${bucketName}' in new project...`)
+ const { error } = await newSupabase.storage.createBucket(bucketName, options)
+ if (error) throw new Error(`❌ Failed to create bucket '${bucketName}': ${error.message}`)
+ console.log(`✅ Created bucket '${bucketName}'`)
+ } else {
+ console.log(`ℹ️ Bucket '${bucketName}' already exists in new project`)
+ }
+}
+
+/**
+ * Migrates a single file from the old project to the new one.
+ */
+async function migrateFile(sourceBucketName, targetBucketName, file) {
+ const loader = createLoadingAnimation(
+ `Migrating ${file.fullPath} in bucket '${sourceBucketName}' to '${targetBucketName}'...`
+ )
+
+ try {
+ const { data, error: downloadError } = await oldSupabase.storage
+ .from(sourceBucketName)
+ .download(file.fullPath)
+ if (downloadError) {
+ loader.stop(`Failed to migrate ${file.fullPath}: Download error`)
+ throw new Error(`Download failed: ${downloadError.message}`)
+ }
+
+ // Preserve all available metadata from the original file
+ const uploadOptions = {
+ upsert: true,
+ contentType: file.metadata?.mimetype,
+ cacheControl: file.metadata?.cacheControl,
+ }
+
+ const { error: uploadError } = await newSupabase.storage
+ .from(targetBucketName)
+ .upload(file.fullPath, data, uploadOptions)
+ if (uploadError) {
+ loader.stop(`Failed to migrate ${file.fullPath}: Upload error`)
+ throw new Error(`Upload failed: ${uploadError.message}`)
+ }
+
+ loader.stop(
+ `Migrated ${file.fullPath} in bucket '${sourceBucketName}' to '${targetBucketName}'`
+ )
+ return { success: true, path: file.fullPath }
+ } catch (err) {
+ console.error(
+ `❌ Error migrating ${file.fullPath} in bucket '${targetBucketName}':`,
+ err.message
+ )
+ return { success: false, path: file.fullPath, error: err.message }
+ }
+}
+
+function chunkArray(array, size) {
+ const chunks = []
+ for (let i = 0; i < array.length; i += size) {
+ chunks.push(array.slice(i, i + size))
+ }
+ return chunks
+}
+
+/**
+ * Migrates all buckets and files from the old Supabase project to the new one.
+ * Processes files in parallel within batches for efficiency.
+ */
+async function migrateBuckets() {
+ console.log('🔄 Starting Supabase Storage migration...')
+ console.log(`📦 Source project: ${OLD_PROJECT_URL}`)
+ console.log(`📦 Target project: ${NEW_PROJECT_URL}`)
+
+ const readline = require('readline').createInterface({
+ input: process.stdin,
+ output: process.stdout,
+ })
+
+ console.log(
+ '\n⚠️ WARNING: This migration may overwrite files in the target project if they have the same paths.'
+ )
+ console.log('⚠️ It is recommended to back up your target project before proceeding.')
+
+ const answer = await new Promise((resolve) => {
+ readline.question('Do you want to proceed with the migration? (yes/no): ', resolve)
+ })
+
+ readline.close()
+
+ if (answer.toLowerCase() !== 'yes') {
+ console.log('Migration canceled by user.')
+ return { canceled: true }
+ }
+
+ console.log('\n📦 Fetching all buckets from old project...')
+
+ const { data: oldBuckets, error: bucketListError } = await oldSupabase.storage.listBuckets()
+
+ if (bucketListError) throw new Error(`❌ Error fetching buckets: ${bucketListError.message}`)
+ console.log(`✅ Found ${oldBuckets.length} buckets to migrate.`)
+
+ const { data: existingBuckets, error: existingBucketsError } =
+ await newSupabase.storage.listBuckets()
+ if (existingBucketsError)
+ throw new Error(`❌ Error fetching existing buckets: ${existingBucketsError.message}`)
+
+ const existingBucketNames = existingBuckets.map((b) => b.name)
+ const conflictingBuckets = oldBuckets.filter((b) => existingBucketNames.includes(b.name))
+
+ let conflictStrategy = 2
+
+ if (conflictingBuckets.length > 0) {
+ console.log('\n⚠️ The following buckets already exist in the target project:')
+ conflictingBuckets.forEach((b) => console.log(` - ${b.name}`))
+
+ const conflictAnswer = await new Promise((resolve) => {
+ const rl = require('readline').createInterface({
+ input: process.stdin,
+ output: process.stdout,
+ })
+ rl.question(
+ '\nHow do you want to handle existing buckets?\n' +
+ '1. Skip existing buckets\n' +
+ '2. Merge files (may overwrite existing files)\n' +
+ '3. Rename buckets in target (add suffix "_migrated")\n' +
+ '4. Cancel migration\n' +
+ 'Enter your choice (1-4): ',
+ (answer) => {
+ rl.close()
+ resolve(answer)
+ }
+ )
+ })
+
+ if (conflictAnswer === '4') {
+ console.log('Migration canceled by user.')
+ return { canceled: true }
+ }
+
+ conflictStrategy = parseInt(conflictAnswer)
+ if (isNaN(conflictStrategy) || conflictStrategy < 1 || conflictStrategy > 3) {
+ console.log('Invalid choice. Migration canceled.')
+ return { canceled: true }
+ }
+ }
+
+ const migrationStats = {
+ totalBuckets: oldBuckets.length,
+ processedBuckets: 0,
+ skippedBuckets: 0,
+ totalFiles: 0,
+ successfulFiles: 0,
+ failedFiles: 0,
+ failedFilesList: [],
+ }
+
+ for (const bucket of oldBuckets) {
+ const bucketName = bucket.name
+ console.log(`\n📁 Processing bucket: ${bucketName}`)
+
+ let targetBucketName = bucketName
+
+ if (existingBucketNames.includes(bucketName)) {
+ if (conflictStrategy === 1) {
+ console.log(`⏩ Skipping bucket '${bucketName}' as it already exists in target project`)
+ migrationStats.skippedBuckets++
+ continue
+ } else if (conflictStrategy === 3) {
+ targetBucketName = `${bucketName}_migrated`
+ console.log(`🔄 Renaming bucket to '${targetBucketName}' in target project`)
+ } else {
+ console.log(`🔄 Merging files into existing bucket '${bucketName}' in target project`)
}
- } catch (err) {
- console.log('error moving ', objectData)
- console.log(err)
}
+
+ // Preserve bucket configuration when creating in the new project
+ if (targetBucketName !== bucketName || !existingBucketNames.includes(bucketName)) {
+ await ensureBucketExists(targetBucketName, {
+ public: bucket.public,
+ fileSizeLimit: bucket.file_size_limit,
+ allowedMimeTypes: bucket.allowed_mime_types,
+ })
+ }
+
+ const files = await listAllFiles(bucketName)
+ console.log(`✅ Found ${files.length} files in bucket '${bucketName}'.`)
+ migrationStats.totalFiles += files.length
+
+ const batches = chunkArray(files, 10)
+
+ for (let i = 0; i < batches.length; i++) {
+ console.log(`\n🚀 Processing batch ${i + 1}/${batches.length} (${batches[i].length} files)`)
+
+ const results = await Promise.all(
+ batches[i].map((file) => migrateFile(bucketName, targetBucketName, file))
+ )
+
+ const batchSuccesses = results.filter((r) => r.success).length
+ const batchFailures = results.filter((r) => !r.success)
+
+ migrationStats.successfulFiles += batchSuccesses
+ migrationStats.failedFiles += batchFailures.length
+ migrationStats.failedFilesList.push(...batchFailures.map((f) => f.path))
+
+ console.log(
+ `✅ Completed batch ${i + 1}/${batches.length}: ${batchSuccesses} succeeded, ${batchFailures.length} failed`
+ )
+ }
+
+ migrationStats.processedBuckets++
+ console.log(`✅ Completed bucket '${bucketName}' migration`)
+ }
+
+ console.log('\n📊 Migration Summary:')
+ console.log(
+ `Buckets: ${migrationStats.processedBuckets}/${migrationStats.totalBuckets} processed, ${migrationStats.skippedBuckets} skipped`
+ )
+ console.log(
+ `Files: ${migrationStats.successfulFiles} succeeded, ${migrationStats.failedFiles} failed (${migrationStats.totalFiles} total)`
+ )
+
+ if (migrationStats.failedFiles > 0) {
+ console.log('\n⚠️ Failed files:')
+ migrationStats.failedFilesList.forEach((path) => console.log(` - ${path}`))
+ return migrationStats
}
-})()
+
+ return migrationStats
+}
+
+migrateBuckets()
+ .then((stats) => {
+ if (stats.failedFiles > 0) {
+ console.log(`\n⚠️ Migration completed with ${stats.failedFiles} failed files.`)
+ process.exit(1)
+ } else {
+ console.log('\n🎉 Migration completed successfully!')
+ process.exit(0)
+ }
+ })
+ .catch((err) => {
+ console.error('❌ Fatal error during migration:', err.message)
+ process.exit(1)
+ })
```