Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(backend): resolve panic in schema synchronization (#18159)
Danny Xuf21ac013838e
diff --git a/backend/runner/schemasync/syncer.go b/backend/runner/schemasync/syncer.goindex 5b5d17f7d1..378d0d6ca7 100644--- a/backend/runner/schemasync/syncer.go+++ b/backend/runner/schemasync/syncer.go@@ -24,7 +24,6 @@ import ("github.com/bytebase/bytebase/backend/component/state""github.com/bytebase/bytebase/backend/enterprise"storepb "github.com/bytebase/bytebase/backend/generated-go/store"- v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db""github.com/bytebase/bytebase/backend/store""github.com/bytebase/bytebase/backend/store/model"@@ -369,12 +368,12 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Datab// Sync database schemadeadlineCtx, cancelFunc := context.WithDeadline(ctx, time.Now().Add(syncTimeout))defer cancelFunc()- databaseMetadata, err := driver.SyncDBSchema(deadlineCtx)+ syncedDatabaseMetadata, err := driver.SyncDBSchema(deadlineCtx)if err != nil {return 0, errors.Wrapf(err, "failed to sync database schema for database %q", database.DatabaseName)}var schemaBuf bytes.Buffer- if err := driver.Dump(deadlineCtx, &schemaBuf, databaseMetadata); err != nil {+ if err := driver.Dump(deadlineCtx, &schemaBuf, syncedDatabaseMetadata); err != nil {return 0, errors.Wrapf(err, "failed to dump database schema for database %q", database.DatabaseName)}rawDump := schemaBuf.Bytes()@@ -386,6 +385,11 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Databif err != nil {return 0, errors.Wrapf(err, "failed to get database schema for database %q", database.DatabaseName)}+ // If the schema does not exist, then we create a new one.+ // This happens when creating a new database in the test.+ if dbMetadata == nil {+ dbMetadata = model.NewDatabaseMetadata(&storepb.DatabaseSchemaMetadata{}, nil, &storepb.DatabaseConfig{}, instance.Metadata.GetEngine(), store.IsObjectCaseSensitive(instance))+ }project, err := s.store.GetProjectV2(ctx, &store.FindProjectMessage{ResourceID: &database.ProjectID,@@ -402,18 +406,14 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Datab// Force to disable classification from comment if the engine is not MYSQL or PG.classificationConfig.ClassificationFromConfig = true}- var dbConfig *storepb.DatabaseConfig++ dbConfig := dbMetadata.GetConfig()if classificationConfig.ClassificationFromConfig {- // Only set the user comment.- setUserCommentFromComment(databaseMetadata)- dbConfig = dbMetadata.BuildDatabaseConfig()+ // Only set the user comment from the database comment.+ setUserCommentFromComment(syncedDatabaseMetadata)} else {- // Get classification from the comment.- if err := s.licenseService.IsFeatureEnabledForInstance(v1pb.PlanFeature_FEATURE_DATA_CLASSIFICATION, instance); err == nil {- dbConfig = buildDatabaseConfigWithClassificationFromComment(databaseMetadata, dbMetadata, classificationConfig)- } else {- dbConfig = dbMetadata.BuildDatabaseConfig()- }+ // Extract classification from comments and update the config in place.+ dbConfig = updateClassificationFromComment(syncedDatabaseMetadata, dbMetadata, classificationConfig)}// Check for schema drift only when not creating sync history@@ -429,8 +429,8 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.DatabmetadataUpdates := []func(*storepb.DatabaseMetadata){func(md *storepb.DatabaseMetadata) {md.LastSyncTime = timestamppb.Now()- md.BackupAvailable = s.databaseBackupAvailable(ctx, instance, databaseMetadata)- md.Datashare = databaseMetadata.Datashare+ md.BackupAvailable = s.databaseBackupAvailable(ctx, instance, syncedDatabaseMetadata)+ md.Datashare = syncedDatabaseMetadata.Datashareif !createSyncHistory {md.Drifted = !skipped && drifted}@@ -448,10 +448,10 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Databif err := s.store.UpsertDBSchema(ctx,database.InstanceID, database.DatabaseName,- databaseMetadata, dbConfig, rawDump,+ syncedDatabaseMetadata, dbConfig, rawDump,); err != nil {if strings.Contains(err.Error(), "escape sequence") {- if metadataBytes, err := protojson.Marshal(databaseMetadata); err == nil {+ if metadataBytes, err := protojson.Marshal(syncedDatabaseMetadata); err == nil {slog.Error("unsupported Unicode escape sequence", slog.String("metadata", string(metadataBytes)), slog.String("raw_dump", string(rawDump)))}}@@ -460,10 +460,10 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Datab// Create sync history if requestedif createSyncHistory {- id, err := s.store.CreateSyncHistory(ctx, database.InstanceID, database.DatabaseName, databaseMetadata, string(rawDump))+ id, err := s.store.CreateSyncHistory(ctx, database.InstanceID, database.DatabaseName, syncedDatabaseMetadata, string(rawDump))if err != nil {if strings.Contains(err.Error(), "escape sequence") {- if metadataBytes, err := protojson.Marshal(databaseMetadata); err == nil {+ if metadataBytes, err := protojson.Marshal(syncedDatabaseMetadata); err == nil {slog.Error("unsupported Unicode escape sequence", slog.String("metadata", string(metadataBytes)), slog.String("raw_dump", string(rawDump)))}}@@ -562,92 +562,42 @@ func (s *Syncer) databaseBackupAvailable(ctx context.Context, instance *store.Inreturn false}-func buildDatabaseConfigWithClassificationFromComment(dbMetadata *storepb.DatabaseSchemaMetadata, databaseConfig *model.DatabaseMetadata, classificationConfig *storepb.DataClassificationSetting_DataClassificationConfig) *storepb.DatabaseConfig {- // Create a new DatabaseConfig to build from scratch- newConfig := &storepb.DatabaseConfig{- Name: dbMetadata.Name,- }+func updateClassificationFromComment(syncedDatabaseMetadata *storepb.DatabaseSchemaMetadata, dbMetadata *model.DatabaseMetadata, classificationConfig *storepb.DataClassificationSetting_DataClassificationConfig) *storepb.DatabaseConfig {+ dbConfig := &storepb.DatabaseConfig{Name: syncedDatabaseMetadata.Name}- for _, schema := range dbMetadata.Schemas {- var tables []*storepb.TableCatalog- hasSchemaContent := false-- // Get existing schema metadata to preserve SemanticType and Labels- var schemaMetadata *model.SchemaMetadata- if databaseConfig != nil {- schemaMetadata = databaseConfig.GetSchemaMetadata(schema.Name)- }+ for _, schema := range syncedDatabaseMetadata.Schemas {+ schemaCatalog := &storepb.SchemaCatalog{Name: schema.Name}+ existSchema := dbMetadata.GetSchemaMetadata(schema.Name)for _, table := range schema.Tables {classification, userComment := common.GetClassificationAndUserComment(table.Comment, classificationConfig)-table.UserComment = userComment- // Get existing table metadata and catalog- var tableMetadata *model.TableMetadata- if schemaMetadata != nil {- tableMetadata = schemaMetadata.GetTable(table.Name)- }-- var columns []*storepb.ColumnCatalog- hasTableContent := false-- // Check if table has classification- if classification != "" {- hasTableContent = true+ tableCatalog := &storepb.TableCatalog{Name: table.Name, Classification: classification}+ existTable := existSchema.GetTable(table.Name)+ if existTable != nil {+ if existTableCatalog := existTable.GetCatalog(); existTableCatalog != nil {+ tableCatalog.Classification = existTableCatalog.Classification+ }}-for _, col := range table.Columns {colClassification, colUserComment := common.GetClassificationAndUserComment(col.Comment, classificationConfig)-col.UserComment = colUserComment- // Get existing column config to preserve SemanticType and Labels- var existingColumnConfig *storepb.ColumnCatalog- if tableMetadata != nil {- if columnMetadata := tableMetadata.GetColumn(col.Name); columnMetadata != nil {- existingColumnConfig = columnMetadata.GetCatalog()+ columnCatalog := &storepb.ColumnCatalog{Name: col.Name, Classification: colClassification}+ if existColumn := existTable.GetColumn(col.Name); existColumn != nil {+ if existColumnCatalog := existColumn.GetCatalog(); existColumnCatalog != nil {+ columnCatalog.SemanticType = existColumnCatalog.SemanticType+ columnCatalog.Labels = existColumnCatalog.Labels}}- if existingColumnConfig == nil {- existingColumnConfig = &storepb.ColumnCatalog{}- }-- // Add column config if it has any meaningful data- if colClassification != "" || existingColumnConfig.GetSemanticType() != "" || len(existingColumnConfig.GetLabels()) > 0 {- columns = append(columns, &storepb.ColumnCatalog{- Name: col.Name,- Classification: colClassification,- SemanticType: existingColumnConfig.SemanticType,- Labels: existingColumnConfig.Labels,- })- hasTableContent = true- }- }-- // Only add table catalog if it has content (either table classification or column configurations)- if hasTableContent {- tableCatalog := &storepb.TableCatalog{- Name: table.Name,- Classification: classification,- Columns: columns,- }- tables = append(tables, tableCatalog)- hasSchemaContent = true- }- }-- // Only add schema catalog if it has content- if hasSchemaContent {- schemaCatalog := &storepb.SchemaCatalog{- Name: schema.Name,- Tables: tables,+ tableCatalog.Columns = append(tableCatalog.Columns, columnCatalog)}- newConfig.Schemas = append(newConfig.Schemas, schemaCatalog)+ schemaCatalog.Tables = append(schemaCatalog.Tables, tableCatalog)}+ dbConfig.Schemas = append(dbConfig.Schemas, schemaCatalog)}-- return newConfig+ return dbConfig}func setUserCommentFromComment(dbMetadata *storepb.DatabaseSchemaMetadata) {