Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: prevent migration 3.13.21 timeout on large audit_log tables (#18701)
Danny Xu5c5abc150fac
diff --git a/backend/migrator/migration/3.13/0021##migrate_user_id_to_email.sql b/backend/migrator/migration/3.13/0021##migrate_user_id_to_email.sqlindex c0315fc1ce..7a2e0b8dc5 100644--- a/backend/migrator/migration/3.13/0021##migrate_user_id_to_email.sql+++ b/backend/migrator/migration/3.13/0021##migrate_user_id_to_email.sql@@ -1,9 +1,11 @@--- OPTIMIZED VERSION-- Migrate user references from users/{id} format to users/{email} format---- Key optimizations:-- 1. Pre-build id->email lookup table (eliminates correlated subqueries)-- 2. O(1) hash lookups instead of O(n×m) correlated subqueries+--+-- NOTE: The audit_log table migration is handled in Go code (migrator.go)+-- to support batching and avoid timeout issues on large datasets.-- ============================================================================-- Build temporary lookup table for O(1) id->email conversion@@ -222,26 +224,5 @@ WHERE payload->'approval' ? 'approvers'WHERE approver ? 'principalId');--- ============================================================================--- 6. Update audit_log table - User field--- ============================================================================-UPDATE audit_log-SET payload = (- SELECT jsonb_set(- audit_log.payload,- '{user}',- to_jsonb(COALESCE(- (- SELECT u.email_format- FROM user_id_to_email u- WHERE u.id = CAST(SUBSTRING(audit_log.payload->>'user' FROM 7) AS INTEGER)- ),- audit_log.payload->>'user'- ))- )-)-WHERE payload->>'user' LIKE 'users/%'- AND payload->>'user' NOT LIKE 'users/%@%';--- Cleanup temp tableDROP TABLE user_id_to_email;diff --git a/backend/migrator/migrator.go b/backend/migrator/migrator.goindex edba0b7a32..f6387aa21d 100644--- a/backend/migrator/migrator.go+++ b/backend/migrator/migrator.go@@ -22,6 +22,20 @@ const latestSchemaFileName = "migration/LATEST.sql"//go:embed migrationvar migrationFS embed.FS+// GoMigrationFunc is a function that performs data migration in Go code.+// It receives a context and database connection, and should handle its own+// transaction management for optimal performance (e.g., batching large updates).+type GoMigrationFunc func(ctx context.Context, conn *sql.Conn) error++// goMigrations is a registry of version-specific Go migrations that run+// in addition to or instead of SQL migrations. These are useful for:+// - Large data migrations that need batching+// - Complex transformations better expressed in Go+// - Operations that benefit from programmatic control+var goMigrations = map[string]GoMigrationFunc{+ "3.13.21": migrate3_13_21,+}+// MigrateSchema migrates the schema for metadata database.func MigrateSchema(ctx context.Context, db *sql.DB) error {files, err := getSortedVersionedFiles()@@ -75,6 +89,18 @@ func MigrateSchema(ctx context.Context, db *sql.DB) error {}version := f.version.String()slog.Info(fmt.Sprintf("Migrating %s.", version))++ // Run Go migration FIRST if one exists for this version+ // This ensures that if Go migration fails, the version is not recorded+ // and both SQL and Go migrations will retry on next startup+ if goMigration, exists := goMigrations[version]; exists {+ slog.Info(fmt.Sprintf("Running Go migration for %s.", version))+ if err := goMigration(ctx, conn); err != nil {+ return errors.Wrapf(err, "Go migration %s failed", version)+ }+ }++ // Run SQL migration, which records the version upon successif err := executeMigration(ctx, conn, string(buf), version); err != nil {return err}@@ -208,3 +234,158 @@ func getLatestDatabaseVersion(ctx context.Context, conn *sql.Conn) (*semver.Vers}return &version, nil}++// migrate3_13_21 migrates audit_log user references from users/{id} to users/{email} format.+// This is done in Go with batching to handle large audit_log tables efficiently.+func migrate3_13_21(ctx context.Context, conn *sql.Conn) error {+ const batchSize = 10000++ // Build user ID to email lookup map+ userMap, err := buildUserIDToEmailMap(ctx, conn)+ if err != nil {+ return errors.Wrap(err, "failed to build user ID to email map")+ }++ var lastID int64+ totalUpdated := 0+ batchCount := 0++ for {+ rowsUpdated, maxID, done, err := migrateAuditLogBatch(ctx, conn, userMap, lastID, batchSize)+ if err != nil {+ return err+ }+ if done {+ break+ }++ totalUpdated += rowsUpdated+ batchCount+++ lastID = maxID++ // Log progress every 10 batches (100k rows)+ if batchCount%10 == 0 {+ slog.Info(fmt.Sprintf("Updated %d audit_log rows...", totalUpdated))+ }+ }++ slog.Info(fmt.Sprintf("Completed audit_log migration. Total rows updated: %d", totalUpdated))+ return nil+}++// migrateAuditLogBatch processes a single batch of audit_log updates.+func migrateAuditLogBatch(ctx context.Context, conn *sql.Conn, userMap map[int]string, lastID int64, batchSize int) (rowsUpdated int, maxID int64, done bool, err error) {+ txn, err := conn.BeginTx(ctx, nil)+ if err != nil {+ return 0, 0, false, errors.Wrap(err, "failed to begin transaction")+ }+ defer txn.Rollback()++ // Get batch of rows to update+ rows, err := txn.QueryContext(ctx, `+ SELECT id, payload->>'user' AS user_ref+ FROM audit_log+ WHERE id > $1+ AND payload->>'user' LIKE 'users/%'+ AND payload->>'user' NOT LIKE 'users/%@%'+ ORDER BY id+ LIMIT $2+ `, lastID, batchSize)+ if err != nil {+ return 0, 0, false, errors.Wrap(err, "failed to query batch")+ }+ defer rows.Close()++ // Build VALUES clause for bulk update+ var valueStrings []string+ var valueArgs []any+ argPos := 1++ for rows.Next() {+ var id int64+ var userRef string+ if err := rows.Scan(&id, &userRef); err != nil {+ return 0, 0, false, errors.Wrap(err, "failed to scan row")+ }++ newUserRef := convertUserIDToEmail(userRef, userMap)+ valueStrings = append(valueStrings, fmt.Sprintf("($%d, $%d)", argPos, argPos+1))+ valueArgs = append(valueArgs, id, newUserRef)+ argPos += 2+ maxID = id+ }+ if err := rows.Err(); err != nil {+ return 0, 0, false, errors.Wrap(err, "failed to iterate rows")+ }++ // If no rows, we're done+ if len(valueStrings) == 0 {+ return 0, 0, true, nil+ }++ // Execute bulk update: UPDATE audit_log SET ... FROM (VALUES ...) AS v(id, new_user) WHERE audit_log.id = v.id+ updateSQL := fmt.Sprintf(`+ UPDATE audit_log+ SET payload = jsonb_set(payload, '{user}', to_jsonb(v.new_user))+ FROM (VALUES %s) AS v(id, new_user)+ WHERE audit_log.id = v.id+ `, strings.Join(valueStrings, ", "))++ result, err := txn.ExecContext(ctx, updateSQL, valueArgs...)+ if err != nil {+ return 0, 0, false, errors.Wrap(err, "failed to execute batch update")+ }++ rowsAffected, _ := result.RowsAffected()++ // Commit this batch+ if err := txn.Commit(); err != nil {+ return 0, 0, false, errors.Wrap(err, "failed to commit batch")+ }++ return int(rowsAffected), maxID, false, nil+}++// buildUserIDToEmailMap creates a lookup map from user ID to email format.+func buildUserIDToEmailMap(ctx context.Context, conn *sql.Conn) (map[int]string, error) {+ rows, err := conn.QueryContext(ctx, `SELECT id, email FROM principal WHERE email IS NOT NULL`)+ if err != nil {+ return nil, err+ }+ defer rows.Close()++ userMap := make(map[int]string)+ for rows.Next() {+ var id int+ var email string+ if err := rows.Scan(&id, &email); err != nil {+ return nil, err+ }+ userMap[id] = email+ }+ return userMap, rows.Err()+}++// convertUserIDToEmail converts a user reference from users/{id} to users/{email} format.+func convertUserIDToEmail(userRef string, userMap map[int]string) string {+ // Already in email format+ if !strings.HasPrefix(userRef, "users/") || strings.Contains(userRef, "@") {+ return userRef+ }++ // Extract ID from users/{id}+ idStr := strings.TrimPrefix(userRef, "users/")+ id, err := strconv.Atoi(idStr)+ if err != nil {+ // Can't parse as int, return original+ return userRef+ }++ // Look up email+ if email, found := userMap[id]; found {+ return fmt.Sprintf("users/%s", email)+ }++ // Not found, return original+ return userRef+}