Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
perf: optimize engine filtering and RevisionService latency (#18562)
Danny Xucf6deaa4d058
diff --git a/backend/api/v1/database_service.go b/backend/api/v1/database_service.goindex f311c3af63..6b4d6113e0 100644--- a/backend/api/v1/database_service.go+++ b/backend/api/v1/database_service.go@@ -629,14 +629,6 @@ func (s *DatabaseService) GetDatabaseSDLSchema(ctx context.Context, req *connectreturn nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("%v", err.Error()))}- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &instanceID})- if err != nil {- return nil, errors.Wrapf(err, "failed to get instance %s", instanceID)- }- if instance == nil {- return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("instance %q not found", instanceID))- }-database, err := s.store.GetDatabase(ctx, &store.FindDatabaseMessage{InstanceID: &instanceID,DatabaseName: &databaseName,@@ -647,10 +639,9 @@ func (s *DatabaseService) GetDatabaseSDLSchema(ctx context.Context, req *connectif database == nil {return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("database %q not found", databaseName))}-dbMetadata, err := s.store.GetDBSchema(ctx, &store.FindDBSchemaMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,+ InstanceID: instanceID,+ DatabaseName: databaseName,})if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("%v", err.Error()))@@ -684,9 +675,9 @@ func (s *DatabaseService) GetDatabaseSDLSchema(ctx context.Context, req *connectswitch format {case v1pb.GetDatabaseSDLSchemaRequest_SINGLE_FILE:- return s.getSingleFileSDL(ctx, instance.Metadata.Engine, metadata)+ return s.getSingleFileSDL(ctx, database.Engine, metadata)case v1pb.GetDatabaseSDLSchemaRequest_MULTI_FILE:- return s.getMultiFileSDL(ctx, instance.Metadata.Engine, metadata, databaseName)+ return s.getMultiFileSDL(ctx, database.Engine, metadata, databaseName)default:return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported SDL format: %v", format))}@@ -996,15 +987,6 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requif database == nil {return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("database %q not found", req.Msg.Name))}-- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &database.InstanceID})- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("%v", err.Error()))- }- if instance == nil {- return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("instance %q not found", database.InstanceID))- }-dbMetadata, err := s.store.GetDBSchema(ctx, &store.FindDBSchemaMessage{InstanceID: instanceID,DatabaseName: databaseName,@@ -1019,7 +1001,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requreturn nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("metadata is required"))}storeSchema := convertV1DatabaseMetadata(req.Msg.Metadata)- s, err := schema.GetDatabaseDefinition(instance.Metadata.Engine, schema.GetDefinitionContext{+ s, err := schema.GetDatabaseDefinition(database.Engine, schema.GetDefinitionContext{SkipBackupSchema: false,PrintHeader: false,}, storeSchema)@@ -1029,7 +1011,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requreturn connect.NewResponse(&v1pb.GetSchemaStringResponse{SchemaString: s}), nilcase v1pb.GetSchemaStringRequest_DATABASE:metadata := dbMetadata.GetProto()- s, err := schema.GetDatabaseDefinition(instance.Metadata.Engine, schema.GetDefinitionContext{+ s, err := schema.GetDatabaseDefinition(database.Engine, schema.GetDefinitionContext{SkipBackupSchema: false,PrintHeader: false,}, metadata)@@ -1043,7 +1025,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requreturn nil, connect.NewError(connect.CodeNotFound, errors.Errorf("schema %q not found", req.Msg.Schema))}- s, err := schema.GetSchemaDefinition(instance.Metadata.Engine, schemaMetadata.GetProto())+ s, err := schema.GetSchemaDefinition(database.Engine, schemaMetadata.GetProto())if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get schema schema: %v", err))}@@ -1058,7 +1040,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requreturn nil, connect.NewError(connect.CodeNotFound, errors.Errorf("table %q not found", req.Msg.Object))}sequences := schemaMetadata.GetSequencesByOwnerTable(req.Msg.Object)- s, err := schema.GetTableDefinition(instance.Metadata.Engine, req.Msg.Schema, tableMetadata.GetProto(), sequences)+ s, err := schema.GetTableDefinition(database.Engine, req.Msg.Schema, tableMetadata.GetProto(), sequences)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get table schema: %v", err))}@@ -1072,7 +1054,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requif viewMetadata == nil {return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("view %q not found", req.Msg.Object))}- s, err := schema.GetViewDefinition(instance.Metadata.Engine, req.Msg.Schema, viewMetadata)+ s, err := schema.GetViewDefinition(database.Engine, req.Msg.Schema, viewMetadata)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get view schema: %v", err))}@@ -1086,7 +1068,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requif materializedViewMetadata == nil {return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("materialized view %q not found", req.Msg.Object))}- s, err := schema.GetMaterializedViewDefinition(instance.Metadata.Engine, req.Msg.Schema, materializedViewMetadata)+ s, err := schema.GetMaterializedViewDefinition(database.Engine, req.Msg.Schema, materializedViewMetadata)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get materialized view schema: %v", err))}@@ -1106,7 +1088,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requif functionMetadata == nil {return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("function %q not found", req.Msg.Object))}- s, err := schema.GetFunctionDefinition(instance.Metadata.Engine, req.Msg.Schema, functionMetadata)+ s, err := schema.GetFunctionDefinition(database.Engine, req.Msg.Schema, functionMetadata)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get function schema: %v", err))}@@ -1120,7 +1102,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requif procedureMetadata == nil {return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("procedure %q not found", req.Msg.Object))}- s, err := schema.GetProcedureDefinition(instance.Metadata.Engine, req.Msg.Schema, procedureMetadata)+ s, err := schema.GetProcedureDefinition(database.Engine, req.Msg.Schema, procedureMetadata)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get procedure schema: %v", err))}@@ -1134,7 +1116,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requif sequenceMetadata == nil {return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("sequence %q not found", req.Msg.Object))}- s, err := schema.GetSequenceDefinition(instance.Metadata.Engine, req.Msg.Schema, sequenceMetadata)+ s, err := schema.GetSequenceDefinition(database.Engine, req.Msg.Schema, sequenceMetadata)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get sequence schema: %v", err))}diff --git a/backend/api/v1/database_service_changelog.go b/backend/api/v1/database_service_changelog.goindex 4140776352..5e66bdb86d 100644--- a/backend/api/v1/database_service_changelog.go+++ b/backend/api/v1/database_service_changelog.go@@ -92,16 +92,6 @@ func (s *DatabaseService) GetChangelog(ctx context.Context, req *connect.Requestreturn nil, connect.NewError(connect.CodeNotFound, errors.Errorf("changelog %q not found", changelogUID))}- // Get related database to convert changelog from store.- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{- ResourceID: &instanceID,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, err)- }- if instance == nil {- return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("instance %q not found", instanceID))- }database, err := s.store.GetDatabase(ctx, &store.FindDatabaseMessage{InstanceID: &instanceID,DatabaseName: &databaseName,diff --git a/backend/api/v1/instance_service.go b/backend/api/v1/instance_service.goindex 36bba63bec..3235fd7ae3 100644--- a/backend/api/v1/instance_service.go+++ b/backend/api/v1/instance_service.go@@ -896,13 +896,6 @@ func (s *InstanceService) RemoveDataSource(ctx context.Context, req *connect.Reqreturn nil, connect.NewError(connect.CodeInternal, err)}- instance, err = s.store.GetInstance(ctx, &store.FindInstanceMessage{- ResourceID: &instance.ResourceID,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, err)- }-result := convertInstanceMessage(instance)return connect.NewResponse(result), nil}diff --git a/backend/api/v1/revision_service.go b/backend/api/v1/revision_service.goindex 99b0675921..d5d0dbf8a9 100644--- a/backend/api/v1/revision_service.go+++ b/backend/api/v1/revision_service.go@@ -83,10 +83,7 @@ func (s *RevisionService) ListRevisions(revisions = revisions[:offset.limit]}- converted, err := convertToRevisions(ctx, s.store, request.Parent, revisions)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert to revisions"))- }+ converted := convertToRevisions(request.Parent, database.ProjectID, revisions)return connect.NewResponse(&v1pb.ListRevisionsResponse{Revisions: converted,@@ -103,15 +100,23 @@ func (s *RevisionService) GetRevision(if err != nil {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Wrapf(err, "failed to get revision UID from %v", request.Name))}+ database, err := s.store.GetDatabase(ctx, &store.FindDatabaseMessage{+ InstanceID: &instanceID,+ DatabaseName: &databaseName,+ })+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database"))+ }+ if database == nil {+ return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("database not found"))+ }+revision, err := s.store.GetRevision(ctx, revisionUID, instanceID, databaseName)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get revision %v", request.Name))}parent := common.FormatDatabase(instanceID, databaseName)- converted, err := convertToRevision(ctx, s.store, parent, revision)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert to revision"))- }+ converted := convertToRevision(parent, database.ProjectID, revision)return connect.NewResponse(converted), nil}@@ -250,10 +255,7 @@ func (s *RevisionService) createRevisions(if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to create revision"))}- converted, err := convertToRevision(ctx, s.store, parent, revisionM)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert to revision"))- }+ converted := convertToRevision(parent, database.ProjectID, revisionM)createdRevisions = append(createdRevisions, converted)}return createdRevisions, nil@@ -278,36 +280,22 @@ func (s *RevisionService) DeleteRevision(return connect.NewResponse(&emptypb.Empty{}), nil}-func convertToRevisions(ctx context.Context, s *store.Store, parent string, revisions []*store.RevisionMessage) ([]*v1pb.Revision, error) {+func convertToRevisions(parent string, projectID string, revisions []*store.RevisionMessage) []*v1pb.Revision {var rs []*v1pb.Revisionfor _, revision := range revisions {- r, err := convertToRevision(ctx, s, parent, revision)- if err != nil {- return nil, errors.Wrapf(err, "failed to convert to revisions")- }+ r := convertToRevision(parent, projectID, revision)rs = append(rs, r)}- return rs, nil+ return rs}-func convertToRevision(ctx context.Context, s *store.Store, parent string, revision *store.RevisionMessage) (*v1pb.Revision, error) {- database, err := s.GetDatabase(ctx, &store.FindDatabaseMessage{- InstanceID: &revision.InstanceID,- DatabaseName: &revision.DatabaseName,- })- if err != nil {- return nil, errors.Wrapf(err, "failed to get database")- }- if database == nil {- return nil, errors.Errorf("database not found")- }-+func convertToRevision(parent string, projectID string, revision *store.RevisionMessage) *v1pb.Revision {taskRunName := revision.Payload.TaskRunr := &v1pb.Revision{Name: fmt.Sprintf("%s/%s%d", parent, common.RevisionNamePrefix, revision.UID),Release: revision.Payload.Release,CreateTime: timestamppb.New(revision.CreatedAt),- Sheet: common.FormatSheet(database.ProjectID, revision.Payload.SheetSha256),+ Sheet: common.FormatSheet(projectID, revision.Payload.SheetSha256),SheetSha256: revision.Payload.SheetSha256,Version: revision.Version,File: revision.Payload.File,@@ -322,7 +310,7 @@ func convertToRevision(ctx context.Context, s *store.Store, parent string, revisr.DeleteTime = timestamppb.New(*revision.DeletedAt)}- return r, nil+ return r}func convertToRevisionType(t storepb.SchemaChangeType) v1pb.Revision_Type {diff --git a/backend/api/v1/rollout_service_converter.go b/backend/api/v1/rollout_service_converter.goindex dd8ce8c51a..7206e82bd0 100644--- a/backend/api/v1/rollout_service_converter.go+++ b/backend/api/v1/rollout_service_converter.go@@ -291,11 +291,11 @@ func convertToTask(ctx context.Context, s *store.Store, project *store.ProjectMereturn convertToTaskFromDatabaseCreate(ctx, s, project, task)case storepb.Task_DATABASE_MIGRATE:// All DATABASE_MIGRATE tasks are treated as schema updates (DDL or GHOST)- return convertToTaskFromSchemaUpdate(ctx, s, project, task)+ return convertToTaskFromSchemaUpdate(project, task)case storepb.Task_DATABASE_SDL:- return convertToTaskFromSchemaUpdate(ctx, s, project, task)+ return convertToTaskFromSchemaUpdate(project, task)case storepb.Task_DATABASE_EXPORT:- return convertToTaskFromDatabaseDataExport(ctx, s, project, task)+ return convertToTaskFromDatabaseDataExport(project, task)case storepb.Task_TASK_TYPE_UNSPECIFIED:return nil, errors.Errorf("task type %v is not supported", task.Type)default:@@ -339,17 +339,10 @@ func convertToTaskFromDatabaseCreate(ctx context.Context, s *store.Store, projecreturn v1pbTask, nil}-func convertToTaskFromSchemaUpdate(ctx context.Context, s *store.Store, project *store.ProjectMessage, task *store.TaskMessage) (*v1pb.Task, error) {+func convertToTaskFromSchemaUpdate(project *store.ProjectMessage, task *store.TaskMessage) (*v1pb.Task, error) {if task.DatabaseName == nil {return nil, errors.Errorf("schema update task database is nil")}- database, err := s.GetDatabase(ctx, &store.FindDatabaseMessage{InstanceID: &task.InstanceID, DatabaseName: task.DatabaseName, ShowDeleted: true})- if err != nil {- return nil, errors.Wrapf(err, "failed to get database")- }- if database == nil {- return nil, errors.Errorf("database not found")- }// Determine DatabaseChangeType based on task typevar databaseChangeType v1pb.DatabaseChangeType@@ -369,7 +362,7 @@ func convertToTaskFromSchemaUpdate(ctx context.Context, s *store.Store, projectType: convertToTaskType(task),Status: convertToTaskStatus(task.LatestTaskRunStatus, task.Payload.GetSkipped()),SkippedReason: task.Payload.GetSkippedReason(),- Target: fmt.Sprintf("%s%s/%s%s", common.InstanceNamePrefix, database.InstanceID, common.DatabaseIDPrefix, database.DatabaseName),+ Target: fmt.Sprintf("%s%s/%s%s", common.InstanceNamePrefix, task.InstanceID, common.DatabaseIDPrefix, *(task.DatabaseName)),Payload: &v1pb.Task_DatabaseUpdate_{DatabaseUpdate: &v1pb.Task_DatabaseUpdate{Sheet: common.FormatSheet(project.ResourceID, task.Payload.GetSheetSha256()),@@ -387,18 +380,12 @@ func convertToTaskFromSchemaUpdate(ctx context.Context, s *store.Store, projectreturn v1pbTask, nil}-func convertToTaskFromDatabaseDataExport(ctx context.Context, s *store.Store, project *store.ProjectMessage, task *store.TaskMessage) (*v1pb.Task, error) {+func convertToTaskFromDatabaseDataExport(project *store.ProjectMessage, task *store.TaskMessage) (*v1pb.Task, error) {if task.DatabaseName == nil {return nil, errors.Errorf("data export task database is nil")}- database, err := s.GetDatabase(ctx, &store.FindDatabaseMessage{InstanceID: &task.InstanceID, DatabaseName: task.DatabaseName, ShowDeleted: true})- if err != nil {- return nil, errors.Wrapf(err, "failed to get database")- }- if database == nil {- return nil, errors.Errorf("database not found")- }- targetDatabaseName := fmt.Sprintf("%s%s/%s%s", common.InstanceNamePrefix, database.InstanceID, common.DatabaseIDPrefix, database.DatabaseName)++ targetDatabaseName := fmt.Sprintf("%s%s/%s%s", common.InstanceNamePrefix, task.InstanceID, common.DatabaseIDPrefix, *(task.DatabaseName))sheet := common.FormatSheet(project.ResourceID, task.Payload.GetSheetSha256())v1pbTaskPayload := v1pb.Task_DatabaseDataExport_{DatabaseDataExport: &v1pb.Task_DatabaseDataExport{diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex e48b47089d..1034f3f24e 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -1385,16 +1385,6 @@ func BuildGetLinkedDatabaseMetadataFunc(storeInstance *store.Store, engine storefunc BuildGetDatabaseMetadataFunc(storeInstance *store.Store) parserbase.GetDatabaseMetadataFunc {return func(ctx context.Context, instanceID, databaseName string) (string, *model.DatabaseMetadata, error) {- database, err := storeInstance.GetDatabase(ctx, &store.FindDatabaseMessage{- InstanceID: &instanceID,- DatabaseName: &databaseName,- })- if err != nil {- return "", nil, err- }- if database == nil {- return "", nil, nil- }databaseMetadata, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{InstanceID: instanceID,DatabaseName: databaseName,diff --git a/backend/api/v1/worksheet_service.go b/backend/api/v1/worksheet_service.goindex 3857b162b9..f45d9c1e25 100644--- a/backend/api/v1/worksheet_service.go+++ b/backend/api/v1/worksheet_service.go@@ -12,7 +12,6 @@ import ("google.golang.org/protobuf/types/known/timestamppb""github.com/bytebase/bytebase/backend/common"- "github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/component/iam"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/generated-go/v1/v1connect"@@ -95,10 +94,7 @@ func (s *WorksheetService) CreateWorksheet(if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create worksheet: %v", err))}- v1pbWorksheet, err := s.convertToAPIWorksheetMessage(ctx, worksheet)- if err != nil {- return nil, err- }+ v1pbWorksheet := convertToAPIWorksheetMessage(worksheet)return connect.NewResponse(v1pbWorksheet), nil}@@ -133,10 +129,7 @@ func (s *WorksheetService) GetWorksheet(return nil, connect.NewError(connect.CodePermissionDenied, errors.Errorf("cannot access worksheet %s", worksheet.Title))}- v1pbWorksheet, err := s.convertToAPIWorksheetMessage(ctx, worksheet)- if err != nil {- return nil, err- }+ v1pbWorksheet := convertToAPIWorksheetMessage(worksheet)return connect.NewResponse(v1pbWorksheet), nil}@@ -182,15 +175,7 @@ func (s *WorksheetService) SearchWorksheets(slog.Warn("cannot access worksheet", slog.String("name", worksheet.Title))continue}- v1pbWorksheet, err := s.convertToAPIWorksheetMessage(ctx, worksheet)- if err != nil {- var connectErr *connect.Error- if errors.As(err, &connectErr) && connectErr.Code() == connect.CodeNotFound {- slog.Debug("failed to found resource for worksheet", log.BBError(err), slog.Int("id", worksheet.UID), slog.String("project", worksheet.ProjectID))- continue- }- return nil, err- }+ v1pbWorksheet := convertToAPIWorksheetMessage(worksheet)v1pbWorksheets = append(v1pbWorksheets, v1pbWorksheet)}return connect.NewResponse(&v1pb.SearchWorksheetsResponse{@@ -299,11 +284,7 @@ func (s *WorksheetService) UpdateWorksheet(if worksheet == nil {return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("worksheet %q not found", request.Worksheet.Name))}- v1pbWorksheet, err := s.convertToAPIWorksheetMessage(ctx, worksheet)- if err != nil {- return nil, err- }-+ v1pbWorksheet := convertToAPIWorksheetMessage(worksheet)return connect.NewResponse(v1pbWorksheet), nil}@@ -526,20 +507,10 @@ func (s *WorksheetService) checkWorksheetPermission(return ok, nil}-func (s *WorksheetService) convertToAPIWorksheetMessage(ctx context.Context, worksheet *store.WorkSheetMessage) (*v1pb.Worksheet, error) {+func convertToAPIWorksheetMessage(worksheet *store.WorkSheetMessage) *v1pb.Worksheet {databaseParent := ""if worksheet.InstanceID != nil && worksheet.DatabaseName != nil {- database, err := s.store.GetDatabase(ctx, &store.FindDatabaseMessage{- ProjectID: &worksheet.ProjectID,- InstanceID: worksheet.InstanceID,- DatabaseName: worksheet.DatabaseName,- })- if err != nil {- slog.Debug("failed to found database for worksheet", log.BBError(err), slog.Int("id", worksheet.UID), slog.String("instance", *worksheet.InstanceID), slog.String("database", *worksheet.DatabaseName))- }- if database != nil {- databaseParent = common.FormatDatabase(database.InstanceID, database.DatabaseName)- }+ databaseParent = common.FormatDatabase(*worksheet.InstanceID, *worksheet.DatabaseName)}visibility := v1pb.Worksheet_VISIBILITY_UNSPECIFIED@@ -553,27 +524,12 @@ func (s *WorksheetService) convertToAPIWorksheetMessage(ctx context.Context, wordefault:// Keep VISIBILITY_UNSPECIFIED}-- creator, err := s.store.GetUserByEmail(ctx, worksheet.Creator)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to get creator: %v", err))- }-- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{- ResourceID: &worksheet.ProjectID,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to get project: %v", err))- }- if project == nil {- return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("project with id %s not found", worksheet.ProjectID))- }return &v1pb.Worksheet{Name: fmt.Sprintf("%s%d", common.WorksheetIDPrefix, worksheet.UID),- Project: common.FormatProject(project.ResourceID),+ Project: common.FormatProject(worksheet.ProjectID),Database: databaseParent,Title: worksheet.Title,- Creator: fmt.Sprintf("users/%s", creator.Email),+ Creator: fmt.Sprintf("users/%s", worksheet.Creator),CreateTime: timestamppb.New(worksheet.CreatedAt),UpdateTime: timestamppb.New(worksheet.UpdatedAt),Content: []byte(worksheet.Statement),@@ -581,7 +537,7 @@ func (s *WorksheetService) convertToAPIWorksheetMessage(ctx context.Context, worVisibility: visibility,Starred: worksheet.Starred,Folders: worksheet.Folders,- }, nil+ }}func convertToStoreWorksheetMessage(project *store.ProjectMessage, database *store.DatabaseMessage, creator string, worksheet *v1pb.Worksheet) (*store.WorkSheetMessage, error) {diff --git a/backend/migrator/migration/3.14/0002##add_idx_instance_metadata_engine.sql b/backend/migrator/migration/3.14/0002##add_idx_instance_metadata_engine.sqlnew file mode 100644index 0000000000..ffb75a426e--- /dev/null+++ b/backend/migrator/migration/3.14/0002##add_idx_instance_metadata_engine.sql@@ -0,0 +1 @@+CREATE INDEX idx_instance_metadata_engine ON instance((metadata->>'engine'));diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sqlindex a2ce6e2c87..e8ae780cf9 100644--- a/backend/migrator/migration/LATEST.sql+++ b/backend/migrator/migration/LATEST.sql@@ -130,6 +130,8 @@ CREATE TABLE instance (CREATE UNIQUE INDEX idx_instance_unique_resource_id ON instance(resource_id);+CREATE INDEX idx_instance_metadata_engine ON instance((metadata->>'engine'));+ALTER SEQUENCE instance_id_seq RESTART WITH 101;-- db stores the databases for a particular instancediff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.goindex 34d644a46a..f518bec456 100644--- a/backend/migrator/migrator_test.go+++ b/backend/migrator/migrator_test.go@@ -12,7 +12,7 @@ import (func TestLatestVersion(t *testing.T) {files, err := getSortedVersionedFiles()require.NoError(t, err)- require.Equal(t, semver.MustParse("3.14.1"), *files[len(files)-1].version)+ require.Equal(t, semver.MustParse("3.14.2"), *files[len(files)-1].version)}func TestVersionUnique(t *testing.T) {diff --git a/backend/runner/taskrun/database_migrate_executor.go b/backend/runner/taskrun/database_migrate_executor.goindex a836645124..807ffe1386 100644--- a/backend/runner/taskrun/database_migrate_executor.go+++ b/backend/runner/taskrun/database_migrate_executor.go@@ -97,7 +97,7 @@ func (exec *DatabaseMigrateExecutor) runMigrationWithPriorBackup(ctx context.Con})// Check if we should skip backup or not.- if common.EngineSupportPriorBackup(instance.Metadata.GetEngine()) {+ if common.EngineSupportPriorBackup(database.Engine) {var backupErr errorpriorBackupDetail, backupErr = exec.backupData(ctx, driverCtx, statement, task.Payload, task, instance, database)if backupErr != nil {@@ -271,7 +271,7 @@ func (exec *DatabaseMigrateExecutor) backupData(sourceDatabaseName := common.FormatDatabase(database.InstanceID, database.DatabaseName)// Format: instances/{instance}/databases/{database}- backupDBName := common.BackupDatabaseNameOfEngine(instance.Metadata.GetEngine())+ backupDBName := common.BackupDatabaseNameOfEngine(database.Engine)targetDatabaseName := common.FormatDatabase(database.InstanceID, backupDBName)var backupDatabase *store.DatabaseMessagevar backupDriver db.Driver@@ -281,7 +281,7 @@ func (exec *DatabaseMigrateExecutor) backupData(return nil, errors.Wrap(err, "failed to parse backup database")}- if instance.Metadata.GetEngine() != storepb.Engine_POSTGRES {+ if database.Engine != storepb.Engine_POSTGRES {backupDatabase, err = exec.store.GetDatabase(ctx, &store.FindDatabaseMessage{InstanceID: &backupInstanceID, DatabaseName: &backupDatabaseName})if err != nil {return nil, errors.Wrap(err, "failed to get backup database")@@ -315,7 +315,7 @@ func (exec *DatabaseMigrateExecutor) backupData(IsCaseSensitive: store.IsObjectCaseSensitive(instance),DatabaseName: database.DatabaseName,}- if instance.Metadata.GetEngine() == storepb.Engine_ORACLE {+ if database.Engine == storepb.Engine_ORACLE {oracleDriver, ok := driver.(*oracle.Driver)if ok {if version, err := oracleDriver.GetVersion(); err == nil {@@ -329,12 +329,12 @@ func (exec *DatabaseMigrateExecutor) backupData(}prefix := "_" + time.Now().Format("20060102150405")- statements, err := parserbase.TransformDMLToSelect(ctx, instance.Metadata.GetEngine(), tc, originStatement, database.DatabaseName, backupDatabaseName, prefix)+ statements, err := parserbase.TransformDMLToSelect(ctx, database.Engine, tc, originStatement, database.DatabaseName, backupDatabaseName, prefix)if err != nil {return nil, errors.Wrap(err, "failed to transform DML to select")}- prependStatements, err := getPrependStatements(instance.Metadata.GetEngine(), originStatement)+ prependStatements, err := getPrependStatements(database.Engine, originStatement)if err != nil {return nil, errors.Wrap(err, "failed to get prepend statements")}@@ -396,7 +396,7 @@ func (exec *DatabaseMigrateExecutor) backupData(StartPosition: statement.StartPosition,EndPosition: statement.EndPosition,}- if instance.Metadata.GetEngine() == storepb.Engine_POSTGRES {+ if database.Engine == storepb.Engine_POSTGRES {item.TargetTable = &storepb.PriorBackupDetail_Item_Table{Database: sourceDatabaseName,// postgres uses schema as the backup database name currently.@@ -407,7 +407,7 @@ func (exec *DatabaseMigrateExecutor) backupData(priorBackupDetail.Items = append(priorBackupDetail.Items, item)}- if instance.Metadata.GetEngine() != storepb.Engine_POSTGRES {+ if database.Engine != storepb.Engine_POSTGRES {if err := exec.schemaSyncer.SyncDatabaseSchema(ctx, backupDatabase); err != nil {slog.Error("failed to sync backup database schema",slog.String("database", targetDatabaseName),diff --git a/backend/runner/taskrun/executor.go b/backend/runner/taskrun/executor.goindex 74fc69c1cb..d243bcb62d 100644--- a/backend/runner/taskrun/executor.go+++ b/backend/runner/taskrun/executor.go@@ -62,9 +62,8 @@ type migrateContext struct {profile *config.ProfiledbFactory *dbfactory.DBFactory- instance *store.InstanceMessage- database *store.DatabaseMessage- // nullable if type=baseline+ instance *store.InstanceMessage+ database *store.DatabaseMessagesheetSha256 stringtask *store.TaskMessage@@ -141,7 +140,7 @@ func runMigrationWithFunc(// Pre-compute whether schema dump is needed.// Skip dump for pure DML statements (INSERT, UPDATE, DELETE) as they don't change schema.- mc.needDump = computeNeedDump(task.Type, mc.instance.Metadata.GetEngine(), statement)+ mc.needDump = computeNeedDump(task.Type, mc.database.Engine, statement)skipped, err := doMigrationWithFunc(ctx, driverCtx, store, stateCfg, profile, statement, mc, execFunc)if err != nil {@@ -240,7 +239,7 @@ func doMigrationWithFunc(statementRecord, _ := common.TruncateString(statement, common.MaxSheetSize)slog.Debug("Start migration...",- slog.String("instance", instance.ResourceID),+ slog.String("instance", database.InstanceID),slog.String("database", database.DatabaseName),slog.String("type", string(mc.task.Type.String())),slog.String("statement", statementRecord),diff --git a/backend/store/database.go b/backend/store/database.goindex bbb091869f..1ecfa8d704 100644--- a/backend/store/database.go+++ b/backend/store/database.go@@ -30,6 +30,7 @@ type DatabaseMessage struct {Deleted boolMetadata *storepb.DatabaseMetadata+ Engine storepb.Engine}func (d *DatabaseMessage) String() string {@@ -144,7 +145,7 @@ func (s *Store) ListDatabases(ctx context.Context, find *FindDatabaseMessage) ([where.And("db.name = ?", *v)}if v := find.Engine; v != nil {- where.And("instance.metadata->>'engine' = ?", *v)+ where.And("instance.metadata->>'engine' = ?", v.String())}if !find.ShowDeleted {where.And("instance.deleted = ?", false)@@ -162,7 +163,8 @@ func (s *Store) ListDatabases(ctx context.Context, find *FindDatabaseMessage) ([db.instance,db.name,db.deleted,- db.metadata+ db.metadata,+ instance.metadata->>'engine'FROM ?WHERE ?ORDER BY db.project, db.instance, db.name@@ -189,7 +191,7 @@ func (s *Store) ListDatabases(ctx context.Context, find *FindDatabaseMessage) ([for rows.Next() {databaseMessage := &DatabaseMessage{}var metadataString string- var effectiveEnvironment, environment sql.NullString+ var effectiveEnvironment, environment, engine sql.NullStringif err := rows.Scan(&databaseMessage.ProjectID,&effectiveEnvironment,@@ -198,6 +200,7 @@ func (s *Store) ListDatabases(ctx context.Context, find *FindDatabaseMessage) ([&databaseMessage.DatabaseName,&databaseMessage.Deleted,&metadataString,+ &engine,); err != nil {return nil, err}@@ -207,6 +210,11 @@ func (s *Store) ListDatabases(ctx context.Context, find *FindDatabaseMessage) ([if environment.Valid {databaseMessage.EnvironmentID = &environment.String}+ if engine.Valid {+ if v, ok := storepb.Engine_value[engine.String]; ok {+ databaseMessage.Engine = storepb.Engine(v)+ }+ }var metadata storepb.DatabaseMetadataif err := common.ProtojsonUnmarshaler.Unmarshal([]byte(metadataString), &metadata); err != nil {@@ -586,7 +594,7 @@ func GetListDatabaseFilter(filter string) (*qb.Query, error) {return nil, errors.Errorf("invalid engine filter %q", value)}engine := storepb.Engine(engineValue)- return qb.Q().Space("instance.metadata->>'engine' = ?", engine), nil+ return qb.Q().Space("instance.metadata->>'engine' = ?", engine.String()), nilcase "name":return qb.Q().Space("db.name = ?", value), nilcase "drifted":