The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
chore: improve storage migration script (#33927)
Illia Basalaiev9b1377c2ba71
diff --git a/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdx b/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdxindex a012aaa51aa..c4c2275fcb9 100644--- a/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdx+++ b/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdx@@ -236,47 +236,325 @@ const OLD_PROJECT_SERVICE_KEY = 'old-project-service-key-xxx'const NEW_PROJECT_URL = 'https://yyy.supabase.co'const NEW_PROJECT_SERVICE_KEY = 'new-project-service-key-yyy'-;(async () => {- const oldSupabaseRestClient = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY, {- db: {- schema: 'storage',+const oldSupabase = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY)+const newSupabase = createClient(NEW_PROJECT_URL, NEW_PROJECT_SERVICE_KEY)++function createLoadingAnimation(message) {+ const readline = require('readline')+ const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']+ let i = 0+ let timer+ let stopped = false++ const animate = () => {+ if (stopped) return+ process.stdout.write(`\r${frames[i]} ${message}`)+ i = (i + 1) % frames.length+ timer = setTimeout(animate, 80)+ }++ animate()++ return {+ stop: (finalMessage = '') => {+ stopped = true+ clearTimeout(timer)+ readline.clearLine(process.stdout, 0)+ readline.cursorTo(process.stdout, 0)+ process.stdout.write(`✓ ${finalMessage || message}\n`)},- })- const oldSupabaseClient = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY)- const newSupabaseClient = createClient(NEW_PROJECT_URL, NEW_PROJECT_SERVICE_KEY)-- // make sure you update max_rows in postgrest settings if you have a lot of objects- // or paginate here- const { data: oldObjects, error } = await oldSupabaseRestClient.from('objects').select()- if (error) {- console.log('error getting objects from old bucket')- throw error}+}++/**+ * Lists all files in a bucket, handling nested folders recursively.+ */+async function listAllFiles(bucket, path = '') {+ const loader = createLoadingAnimation(`Listing files in '${bucket}${path ? '/' + path : ''}'...`)++ try {+ const { data, error } = await oldSupabase.storage.from(bucket).list(path, { limit: 1000 })+ if (error) {+ loader.stop(`Error listing files in '${bucket}${path ? '/' + path : ''}'`)+ throw new Error(`❌ Error listing files in bucket '${bucket}': ${error.message}`)+ }++ if (!data || data.length === 0) {+ loader.stop(`No files found in '${bucket}${path ? '/' + path : ''}'`)+ return []+ }- for (const objectData of oldObjects) {- console.log(`moving ${objectData.id}`)- try {- const { data, error: downloadObjectError } = await oldSupabaseClient.storage- .from(objectData.bucket_id)- .download(objectData.name)- if (downloadObjectError) {- throw downloadObjectError+ let files = []+ for (const item of data) {+ if (!item.metadata) {+ loader.stop(`Found folder '${item.name}' in '${bucket}${path ? '/' + path : ''}'`)+ const subFiles = await listAllFiles(bucket, `${path}${item.name}/`)+ files = files.concat(subFiles)+ } else {+ files.push({ fullPath: `${path}${item.name}`, metadata: item.metadata })}+ }++ loader.stop(`Found ${files.length} files in '${bucket}${path ? '/' + path : ''}'`)+ return files+ } catch (error) {+ loader.stop()+ throw error+ }+}++/**+ * Creates a bucket in the new Supabase project if it doesn't exist.+ */+async function ensureBucketExists(bucketName, options = {}) {+ const { data: existingBucket, error: getBucketError } =+ await newSupabase.storage.getBucket(bucketName)++ if (getBucketError && !getBucketError.message.includes('not found')) {+ throw new Error(`❌ Error checking if bucket '${bucketName}' exists: ${getBucketError.message}`)+ }- const { _, error: uploadObjectError } = await newSupabaseClient.storage- .from(objectData.bucket_id)- .upload(objectData.name, data, {- upsert: true,- contentType: objectData.metadata.mimetype,- cacheControl: objectData.metadata.cacheControl,- })- if (uploadObjectError) {- throw uploadObjectError+ if (!existingBucket) {+ console.log(`🪣 Creating bucket '${bucketName}' in new project...`)+ const { error } = await newSupabase.storage.createBucket(bucketName, options)+ if (error) throw new Error(`❌ Failed to create bucket '${bucketName}': ${error.message}`)+ console.log(`✅ Created bucket '${bucketName}'`)+ } else {+ console.log(`ℹ️ Bucket '${bucketName}' already exists in new project`)+ }+}++/**+ * Migrates a single file from the old project to the new one.+ */+async function migrateFile(sourceBucketName, targetBucketName, file) {+ const loader = createLoadingAnimation(+ `Migrating ${file.fullPath} in bucket '${sourceBucketName}' to '${targetBucketName}'...`+ )++ try {+ const { data, error: downloadError } = await oldSupabase.storage+ .from(sourceBucketName)+ .download(file.fullPath)+ if (downloadError) {+ loader.stop(`Failed to migrate ${file.fullPath}: Download error`)+ throw new Error(`Download failed: ${downloadError.message}`)+ }++ // Preserve all available metadata from the original file+ const uploadOptions = {+ upsert: true,+ contentType: file.metadata?.mimetype,+ cacheControl: file.metadata?.cacheControl,+ }++ const { error: uploadError } = await newSupabase.storage+ .from(targetBucketName)+ .upload(file.fullPath, data, uploadOptions)+ if (uploadError) {+ loader.stop(`Failed to migrate ${file.fullPath}: Upload error`)+ throw new Error(`Upload failed: ${uploadError.message}`)+ }++ loader.stop(+ `Migrated ${file.fullPath} in bucket '${sourceBucketName}' to '${targetBucketName}'`+ )+ return { success: true, path: file.fullPath }+ } catch (err) {+ console.error(+ `❌ Error migrating ${file.fullPath} in bucket '${targetBucketName}':`,+ err.message+ )+ return { success: false, path: file.fullPath, error: err.message }+ }+}++function chunkArray(array, size) {+ const chunks = []+ for (let i = 0; i < array.length; i += size) {+ chunks.push(array.slice(i, i + size))+ }+ return chunks+}++/**+ * Migrates all buckets and files from the old Supabase project to the new one.+ * Processes files in parallel within batches for efficiency.+ */+async function migrateBuckets() {+ console.log('🔄 Starting Supabase Storage migration...')+ console.log(`📦 Source project: ${OLD_PROJECT_URL}`)+ console.log(`📦 Target project: ${NEW_PROJECT_URL}`)++ const readline = require('readline').createInterface({+ input: process.stdin,+ output: process.stdout,+ })++ console.log(+ '\n⚠️ WARNING: This migration may overwrite files in the target project if they have the same paths.'+ )+ console.log('⚠️ It is recommended to back up your target project before proceeding.')++ const answer = await new Promise((resolve) => {+ readline.question('Do you want to proceed with the migration? (yes/no): ', resolve)+ })++ readline.close()++ if (answer.toLowerCase() !== 'yes') {+ console.log('Migration canceled by user.')+ return { canceled: true }+ }++ console.log('\n📦 Fetching all buckets from old project...')++ const { data: oldBuckets, error: bucketListError } = await oldSupabase.storage.listBuckets()++ if (bucketListError) throw new Error(`❌ Error fetching buckets: ${bucketListError.message}`)+ console.log(`✅ Found ${oldBuckets.length} buckets to migrate.`)++ const { data: existingBuckets, error: existingBucketsError } =+ await newSupabase.storage.listBuckets()+ if (existingBucketsError)+ throw new Error(`❌ Error fetching existing buckets: ${existingBucketsError.message}`)++ const existingBucketNames = existingBuckets.map((b) => b.name)+ const conflictingBuckets = oldBuckets.filter((b) => existingBucketNames.includes(b.name))++ let conflictStrategy = 2++ if (conflictingBuckets.length > 0) {+ console.log('\n⚠️ The following buckets already exist in the target project:')+ conflictingBuckets.forEach((b) => console.log(` - ${b.name}`))++ const conflictAnswer = await new Promise((resolve) => {+ const rl = require('readline').createInterface({+ input: process.stdin,+ output: process.stdout,+ })+ rl.question(+ '\nHow do you want to handle existing buckets?\n' ++ '1. Skip existing buckets\n' ++ '2. Merge files (may overwrite existing files)\n' ++ '3. Rename buckets in target (add suffix "_migrated")\n' ++ '4. Cancel migration\n' ++ 'Enter your choice (1-4): ',+ (answer) => {+ rl.close()+ resolve(answer)+ }+ )+ })++ if (conflictAnswer === '4') {+ console.log('Migration canceled by user.')+ return { canceled: true }+ }++ conflictStrategy = parseInt(conflictAnswer)+ if (isNaN(conflictStrategy) || conflictStrategy < 1 || conflictStrategy > 3) {+ console.log('Invalid choice. Migration canceled.')+ return { canceled: true }+ }+ }++ const migrationStats = {+ totalBuckets: oldBuckets.length,+ processedBuckets: 0,+ skippedBuckets: 0,+ totalFiles: 0,+ successfulFiles: 0,+ failedFiles: 0,+ failedFilesList: [],+ }++ for (const bucket of oldBuckets) {+ const bucketName = bucket.name+ console.log(`\n📁 Processing bucket: ${bucketName}`)++ let targetBucketName = bucketName++ if (existingBucketNames.includes(bucketName)) {+ if (conflictStrategy === 1) {+ console.log(`⏩ Skipping bucket '${bucketName}' as it already exists in target project`)+ migrationStats.skippedBuckets+++ continue+ } else if (conflictStrategy === 3) {+ targetBucketName = `${bucketName}_migrated`+ console.log(`🔄 Renaming bucket to '${targetBucketName}' in target project`)+ } else {+ console.log(`🔄 Merging files into existing bucket '${bucketName}' in target project`)}- } catch (err) {- console.log('error moving ', objectData)- console.log(err)}++ // Preserve bucket configuration when creating in the new project+ if (targetBucketName !== bucketName || !existingBucketNames.includes(bucketName)) {+ await ensureBucketExists(targetBucketName, {+ public: bucket.public,+ fileSizeLimit: bucket.file_size_limit,+ allowedMimeTypes: bucket.allowed_mime_types,+ })+ }++ const files = await listAllFiles(bucketName)+ console.log(`✅ Found ${files.length} files in bucket '${bucketName}'.`)+ migrationStats.totalFiles += files.length++ const batches = chunkArray(files, 10)++ for (let i = 0; i < batches.length; i++) {+ console.log(`\n🚀 Processing batch ${i + 1}/${batches.length} (${batches[i].length} files)`)++ const results = await Promise.all(+ batches[i].map((file) => migrateFile(bucketName, targetBucketName, file))+ )++ const batchSuccesses = results.filter((r) => r.success).length+ const batchFailures = results.filter((r) => !r.success)++ migrationStats.successfulFiles += batchSuccesses+ migrationStats.failedFiles += batchFailures.length+ migrationStats.failedFilesList.push(...batchFailures.map((f) => f.path))++ console.log(+ `✅ Completed batch ${i + 1}/${batches.length}: ${batchSuccesses} succeeded, ${batchFailures.length} failed`+ )+ }++ migrationStats.processedBuckets+++ console.log(`✅ Completed bucket '${bucketName}' migration`)+ }++ console.log('\n📊 Migration Summary:')+ console.log(+ `Buckets: ${migrationStats.processedBuckets}/${migrationStats.totalBuckets} processed, ${migrationStats.skippedBuckets} skipped`+ )+ console.log(+ `Files: ${migrationStats.successfulFiles} succeeded, ${migrationStats.failedFiles} failed (${migrationStats.totalFiles} total)`+ )++ if (migrationStats.failedFiles > 0) {+ console.log('\n⚠️ Failed files:')+ migrationStats.failedFilesList.forEach((path) => console.log(` - ${path}`))+ return migrationStats}-})()++ return migrationStats+}++migrateBuckets()+ .then((stats) => {+ if (stats.failedFiles > 0) {+ console.log(`\n⚠️ Migration completed with ${stats.failedFiles} failed files.`)+ process.exit(1)+ } else {+ console.log('\n🎉 Migration completed successfully!')+ process.exit(0)+ }+ })+ .catch((err) => {+ console.error('❌ Fatal error during migration:', err.message)+ process.exit(1)+ })```