Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

perf: optimize engine filtering and RevisionService latency (#18562)

Danny Xucf6deaa4d058
diff --git a/backend/api/v1/database_service.go b/backend/api/v1/database_service.go
index f311c3af63..6b4d6113e0 100644
--- a/backend/api/v1/database_service.go
+++ b/backend/api/v1/database_service.go
@@ -629,14 +629,6 @@ func (s *DatabaseService) GetDatabaseSDLSchema(ctx context.Context, req *connect
return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("%v", err.Error()))
}
- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &instanceID})
- if err != nil {
- return nil, errors.Wrapf(err, "failed to get instance %s", instanceID)
- }
- if instance == nil {
- return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("instance %q not found", instanceID))
- }
-
database, err := s.store.GetDatabase(ctx, &store.FindDatabaseMessage{
InstanceID: &instanceID,
DatabaseName: &databaseName,
@@ -647,10 +639,9 @@ func (s *DatabaseService) GetDatabaseSDLSchema(ctx context.Context, req *connect
if database == nil {
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("database %q not found", databaseName))
}
-
dbMetadata, err := s.store.GetDBSchema(ctx, &store.FindDBSchemaMessage{
- InstanceID: database.InstanceID,
- DatabaseName: database.DatabaseName,
+ InstanceID: instanceID,
+ DatabaseName: databaseName,
})
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("%v", err.Error()))
@@ -684,9 +675,9 @@ func (s *DatabaseService) GetDatabaseSDLSchema(ctx context.Context, req *connect
switch format {
case v1pb.GetDatabaseSDLSchemaRequest_SINGLE_FILE:
- return s.getSingleFileSDL(ctx, instance.Metadata.Engine, metadata)
+ return s.getSingleFileSDL(ctx, database.Engine, metadata)
case v1pb.GetDatabaseSDLSchemaRequest_MULTI_FILE:
- return s.getMultiFileSDL(ctx, instance.Metadata.Engine, metadata, databaseName)
+ return s.getMultiFileSDL(ctx, database.Engine, metadata, databaseName)
default:
return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported SDL format: %v", format))
}
@@ -996,15 +987,6 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requ
if database == nil {
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("database %q not found", req.Msg.Name))
}
-
- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &database.InstanceID})
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("%v", err.Error()))
- }
- if instance == nil {
- return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("instance %q not found", database.InstanceID))
- }
-
dbMetadata, err := s.store.GetDBSchema(ctx, &store.FindDBSchemaMessage{
InstanceID: instanceID,
DatabaseName: databaseName,
@@ -1019,7 +1001,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requ
return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("metadata is required"))
}
storeSchema := convertV1DatabaseMetadata(req.Msg.Metadata)
- s, err := schema.GetDatabaseDefinition(instance.Metadata.Engine, schema.GetDefinitionContext{
+ s, err := schema.GetDatabaseDefinition(database.Engine, schema.GetDefinitionContext{
SkipBackupSchema: false,
PrintHeader: false,
}, storeSchema)
@@ -1029,7 +1011,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requ
return connect.NewResponse(&v1pb.GetSchemaStringResponse{SchemaString: s}), nil
case v1pb.GetSchemaStringRequest_DATABASE:
metadata := dbMetadata.GetProto()
- s, err := schema.GetDatabaseDefinition(instance.Metadata.Engine, schema.GetDefinitionContext{
+ s, err := schema.GetDatabaseDefinition(database.Engine, schema.GetDefinitionContext{
SkipBackupSchema: false,
PrintHeader: false,
}, metadata)
@@ -1043,7 +1025,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requ
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("schema %q not found", req.Msg.Schema))
}
- s, err := schema.GetSchemaDefinition(instance.Metadata.Engine, schemaMetadata.GetProto())
+ s, err := schema.GetSchemaDefinition(database.Engine, schemaMetadata.GetProto())
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get schema schema: %v", err))
}
@@ -1058,7 +1040,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requ
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("table %q not found", req.Msg.Object))
}
sequences := schemaMetadata.GetSequencesByOwnerTable(req.Msg.Object)
- s, err := schema.GetTableDefinition(instance.Metadata.Engine, req.Msg.Schema, tableMetadata.GetProto(), sequences)
+ s, err := schema.GetTableDefinition(database.Engine, req.Msg.Schema, tableMetadata.GetProto(), sequences)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get table schema: %v", err))
}
@@ -1072,7 +1054,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requ
if viewMetadata == nil {
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("view %q not found", req.Msg.Object))
}
- s, err := schema.GetViewDefinition(instance.Metadata.Engine, req.Msg.Schema, viewMetadata)
+ s, err := schema.GetViewDefinition(database.Engine, req.Msg.Schema, viewMetadata)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get view schema: %v", err))
}
@@ -1086,7 +1068,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requ
if materializedViewMetadata == nil {
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("materialized view %q not found", req.Msg.Object))
}
- s, err := schema.GetMaterializedViewDefinition(instance.Metadata.Engine, req.Msg.Schema, materializedViewMetadata)
+ s, err := schema.GetMaterializedViewDefinition(database.Engine, req.Msg.Schema, materializedViewMetadata)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get materialized view schema: %v", err))
}
@@ -1106,7 +1088,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requ
if functionMetadata == nil {
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("function %q not found", req.Msg.Object))
}
- s, err := schema.GetFunctionDefinition(instance.Metadata.Engine, req.Msg.Schema, functionMetadata)
+ s, err := schema.GetFunctionDefinition(database.Engine, req.Msg.Schema, functionMetadata)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get function schema: %v", err))
}
@@ -1120,7 +1102,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requ
if procedureMetadata == nil {
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("procedure %q not found", req.Msg.Object))
}
- s, err := schema.GetProcedureDefinition(instance.Metadata.Engine, req.Msg.Schema, procedureMetadata)
+ s, err := schema.GetProcedureDefinition(database.Engine, req.Msg.Schema, procedureMetadata)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get procedure schema: %v", err))
}
@@ -1134,7 +1116,7 @@ func (s *DatabaseService) GetSchemaString(ctx context.Context, req *connect.Requ
if sequenceMetadata == nil {
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("sequence %q not found", req.Msg.Object))
}
- s, err := schema.GetSequenceDefinition(instance.Metadata.Engine, req.Msg.Schema, sequenceMetadata)
+ s, err := schema.GetSequenceDefinition(database.Engine, req.Msg.Schema, sequenceMetadata)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("Failed to get sequence schema: %v", err))
}
diff --git a/backend/api/v1/database_service_changelog.go b/backend/api/v1/database_service_changelog.go
index 4140776352..5e66bdb86d 100644
--- a/backend/api/v1/database_service_changelog.go
+++ b/backend/api/v1/database_service_changelog.go
@@ -92,16 +92,6 @@ func (s *DatabaseService) GetChangelog(ctx context.Context, req *connect.Request
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("changelog %q not found", changelogUID))
}
- // Get related database to convert changelog from store.
- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{
- ResourceID: &instanceID,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, err)
- }
- if instance == nil {
- return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("instance %q not found", instanceID))
- }
database, err := s.store.GetDatabase(ctx, &store.FindDatabaseMessage{
InstanceID: &instanceID,
DatabaseName: &databaseName,
diff --git a/backend/api/v1/instance_service.go b/backend/api/v1/instance_service.go
index 36bba63bec..3235fd7ae3 100644
--- a/backend/api/v1/instance_service.go
+++ b/backend/api/v1/instance_service.go
@@ -896,13 +896,6 @@ func (s *InstanceService) RemoveDataSource(ctx context.Context, req *connect.Req
return nil, connect.NewError(connect.CodeInternal, err)
}
- instance, err = s.store.GetInstance(ctx, &store.FindInstanceMessage{
- ResourceID: &instance.ResourceID,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, err)
- }
-
result := convertInstanceMessage(instance)
return connect.NewResponse(result), nil
}
diff --git a/backend/api/v1/revision_service.go b/backend/api/v1/revision_service.go
index 99b0675921..d5d0dbf8a9 100644
--- a/backend/api/v1/revision_service.go
+++ b/backend/api/v1/revision_service.go
@@ -83,10 +83,7 @@ func (s *RevisionService) ListRevisions(
revisions = revisions[:offset.limit]
}
- converted, err := convertToRevisions(ctx, s.store, request.Parent, revisions)
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert to revisions"))
- }
+ converted := convertToRevisions(request.Parent, database.ProjectID, revisions)
return connect.NewResponse(&v1pb.ListRevisionsResponse{
Revisions: converted,
@@ -103,15 +100,23 @@ func (s *RevisionService) GetRevision(
if err != nil {
return nil, connect.NewError(connect.CodeInvalidArgument, errors.Wrapf(err, "failed to get revision UID from %v", request.Name))
}
+ database, err := s.store.GetDatabase(ctx, &store.FindDatabaseMessage{
+ InstanceID: &instanceID,
+ DatabaseName: &databaseName,
+ })
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database"))
+ }
+ if database == nil {
+ return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("database not found"))
+ }
+
revision, err := s.store.GetRevision(ctx, revisionUID, instanceID, databaseName)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get revision %v", request.Name))
}
parent := common.FormatDatabase(instanceID, databaseName)
- converted, err := convertToRevision(ctx, s.store, parent, revision)
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert to revision"))
- }
+ converted := convertToRevision(parent, database.ProjectID, revision)
return connect.NewResponse(converted), nil
}
@@ -250,10 +255,7 @@ func (s *RevisionService) createRevisions(
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to create revision"))
}
- converted, err := convertToRevision(ctx, s.store, parent, revisionM)
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to convert to revision"))
- }
+ converted := convertToRevision(parent, database.ProjectID, revisionM)
createdRevisions = append(createdRevisions, converted)
}
return createdRevisions, nil
@@ -278,36 +280,22 @@ func (s *RevisionService) DeleteRevision(
return connect.NewResponse(&emptypb.Empty{}), nil
}
-func convertToRevisions(ctx context.Context, s *store.Store, parent string, revisions []*store.RevisionMessage) ([]*v1pb.Revision, error) {
+func convertToRevisions(parent string, projectID string, revisions []*store.RevisionMessage) []*v1pb.Revision {
var rs []*v1pb.Revision
for _, revision := range revisions {
- r, err := convertToRevision(ctx, s, parent, revision)
- if err != nil {
- return nil, errors.Wrapf(err, "failed to convert to revisions")
- }
+ r := convertToRevision(parent, projectID, revision)
rs = append(rs, r)
}
- return rs, nil
+ return rs
}
-func convertToRevision(ctx context.Context, s *store.Store, parent string, revision *store.RevisionMessage) (*v1pb.Revision, error) {
- database, err := s.GetDatabase(ctx, &store.FindDatabaseMessage{
- InstanceID: &revision.InstanceID,
- DatabaseName: &revision.DatabaseName,
- })
- if err != nil {
- return nil, errors.Wrapf(err, "failed to get database")
- }
- if database == nil {
- return nil, errors.Errorf("database not found")
- }
-
+func convertToRevision(parent string, projectID string, revision *store.RevisionMessage) *v1pb.Revision {
taskRunName := revision.Payload.TaskRun
r := &v1pb.Revision{
Name: fmt.Sprintf("%s/%s%d", parent, common.RevisionNamePrefix, revision.UID),
Release: revision.Payload.Release,
CreateTime: timestamppb.New(revision.CreatedAt),
- Sheet: common.FormatSheet(database.ProjectID, revision.Payload.SheetSha256),
+ Sheet: common.FormatSheet(projectID, revision.Payload.SheetSha256),
SheetSha256: revision.Payload.SheetSha256,
Version: revision.Version,
File: revision.Payload.File,
@@ -322,7 +310,7 @@ func convertToRevision(ctx context.Context, s *store.Store, parent string, revis
r.DeleteTime = timestamppb.New(*revision.DeletedAt)
}
- return r, nil
+ return r
}
func convertToRevisionType(t storepb.SchemaChangeType) v1pb.Revision_Type {
diff --git a/backend/api/v1/rollout_service_converter.go b/backend/api/v1/rollout_service_converter.go
index dd8ce8c51a..7206e82bd0 100644
--- a/backend/api/v1/rollout_service_converter.go
+++ b/backend/api/v1/rollout_service_converter.go
@@ -291,11 +291,11 @@ func convertToTask(ctx context.Context, s *store.Store, project *store.ProjectMe
return convertToTaskFromDatabaseCreate(ctx, s, project, task)
case storepb.Task_DATABASE_MIGRATE:
// All DATABASE_MIGRATE tasks are treated as schema updates (DDL or GHOST)
- return convertToTaskFromSchemaUpdate(ctx, s, project, task)
+ return convertToTaskFromSchemaUpdate(project, task)
case storepb.Task_DATABASE_SDL:
- return convertToTaskFromSchemaUpdate(ctx, s, project, task)
+ return convertToTaskFromSchemaUpdate(project, task)
case storepb.Task_DATABASE_EXPORT:
- return convertToTaskFromDatabaseDataExport(ctx, s, project, task)
+ return convertToTaskFromDatabaseDataExport(project, task)
case storepb.Task_TASK_TYPE_UNSPECIFIED:
return nil, errors.Errorf("task type %v is not supported", task.Type)
default:
@@ -339,17 +339,10 @@ func convertToTaskFromDatabaseCreate(ctx context.Context, s *store.Store, projec
return v1pbTask, nil
}
-func convertToTaskFromSchemaUpdate(ctx context.Context, s *store.Store, project *store.ProjectMessage, task *store.TaskMessage) (*v1pb.Task, error) {
+func convertToTaskFromSchemaUpdate(project *store.ProjectMessage, task *store.TaskMessage) (*v1pb.Task, error) {
if task.DatabaseName == nil {
return nil, errors.Errorf("schema update task database is nil")
}
- database, err := s.GetDatabase(ctx, &store.FindDatabaseMessage{InstanceID: &task.InstanceID, DatabaseName: task.DatabaseName, ShowDeleted: true})
- if err != nil {
- return nil, errors.Wrapf(err, "failed to get database")
- }
- if database == nil {
- return nil, errors.Errorf("database not found")
- }
// Determine DatabaseChangeType based on task type
var databaseChangeType v1pb.DatabaseChangeType
@@ -369,7 +362,7 @@ func convertToTaskFromSchemaUpdate(ctx context.Context, s *store.Store, project
Type: convertToTaskType(task),
Status: convertToTaskStatus(task.LatestTaskRunStatus, task.Payload.GetSkipped()),
SkippedReason: task.Payload.GetSkippedReason(),
- Target: fmt.Sprintf("%s%s/%s%s", common.InstanceNamePrefix, database.InstanceID, common.DatabaseIDPrefix, database.DatabaseName),
+ Target: fmt.Sprintf("%s%s/%s%s", common.InstanceNamePrefix, task.InstanceID, common.DatabaseIDPrefix, *(task.DatabaseName)),
Payload: &v1pb.Task_DatabaseUpdate_{
DatabaseUpdate: &v1pb.Task_DatabaseUpdate{
Sheet: common.FormatSheet(project.ResourceID, task.Payload.GetSheetSha256()),
@@ -387,18 +380,12 @@ func convertToTaskFromSchemaUpdate(ctx context.Context, s *store.Store, project
return v1pbTask, nil
}
-func convertToTaskFromDatabaseDataExport(ctx context.Context, s *store.Store, project *store.ProjectMessage, task *store.TaskMessage) (*v1pb.Task, error) {
+func convertToTaskFromDatabaseDataExport(project *store.ProjectMessage, task *store.TaskMessage) (*v1pb.Task, error) {
if task.DatabaseName == nil {
return nil, errors.Errorf("data export task database is nil")
}
- database, err := s.GetDatabase(ctx, &store.FindDatabaseMessage{InstanceID: &task.InstanceID, DatabaseName: task.DatabaseName, ShowDeleted: true})
- if err != nil {
- return nil, errors.Wrapf(err, "failed to get database")
- }
- if database == nil {
- return nil, errors.Errorf("database not found")
- }
- targetDatabaseName := fmt.Sprintf("%s%s/%s%s", common.InstanceNamePrefix, database.InstanceID, common.DatabaseIDPrefix, database.DatabaseName)
+
+ targetDatabaseName := fmt.Sprintf("%s%s/%s%s", common.InstanceNamePrefix, task.InstanceID, common.DatabaseIDPrefix, *(task.DatabaseName))
sheet := common.FormatSheet(project.ResourceID, task.Payload.GetSheetSha256())
v1pbTaskPayload := v1pb.Task_DatabaseDataExport_{
DatabaseDataExport: &v1pb.Task_DatabaseDataExport{
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.go
index e48b47089d..1034f3f24e 100644
--- a/backend/api/v1/sql_service.go
+++ b/backend/api/v1/sql_service.go
@@ -1385,16 +1385,6 @@ func BuildGetLinkedDatabaseMetadataFunc(storeInstance *store.Store, engine store
func BuildGetDatabaseMetadataFunc(storeInstance *store.Store) parserbase.GetDatabaseMetadataFunc {
return func(ctx context.Context, instanceID, databaseName string) (string, *model.DatabaseMetadata, error) {
- database, err := storeInstance.GetDatabase(ctx, &store.FindDatabaseMessage{
- InstanceID: &instanceID,
- DatabaseName: &databaseName,
- })
- if err != nil {
- return "", nil, err
- }
- if database == nil {
- return "", nil, nil
- }
databaseMetadata, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
InstanceID: instanceID,
DatabaseName: databaseName,
diff --git a/backend/api/v1/worksheet_service.go b/backend/api/v1/worksheet_service.go
index 3857b162b9..f45d9c1e25 100644
--- a/backend/api/v1/worksheet_service.go
+++ b/backend/api/v1/worksheet_service.go
@@ -12,7 +12,6 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/bytebase/bytebase/backend/common"
- "github.com/bytebase/bytebase/backend/common/log"
"github.com/bytebase/bytebase/backend/component/iam"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/generated-go/v1/v1connect"
@@ -95,10 +94,7 @@ func (s *WorksheetService) CreateWorksheet(
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create worksheet: %v", err))
}
- v1pbWorksheet, err := s.convertToAPIWorksheetMessage(ctx, worksheet)
- if err != nil {
- return nil, err
- }
+ v1pbWorksheet := convertToAPIWorksheetMessage(worksheet)
return connect.NewResponse(v1pbWorksheet), nil
}
@@ -133,10 +129,7 @@ func (s *WorksheetService) GetWorksheet(
return nil, connect.NewError(connect.CodePermissionDenied, errors.Errorf("cannot access worksheet %s", worksheet.Title))
}
- v1pbWorksheet, err := s.convertToAPIWorksheetMessage(ctx, worksheet)
- if err != nil {
- return nil, err
- }
+ v1pbWorksheet := convertToAPIWorksheetMessage(worksheet)
return connect.NewResponse(v1pbWorksheet), nil
}
@@ -182,15 +175,7 @@ func (s *WorksheetService) SearchWorksheets(
slog.Warn("cannot access worksheet", slog.String("name", worksheet.Title))
continue
}
- v1pbWorksheet, err := s.convertToAPIWorksheetMessage(ctx, worksheet)
- if err != nil {
- var connectErr *connect.Error
- if errors.As(err, &connectErr) && connectErr.Code() == connect.CodeNotFound {
- slog.Debug("failed to found resource for worksheet", log.BBError(err), slog.Int("id", worksheet.UID), slog.String("project", worksheet.ProjectID))
- continue
- }
- return nil, err
- }
+ v1pbWorksheet := convertToAPIWorksheetMessage(worksheet)
v1pbWorksheets = append(v1pbWorksheets, v1pbWorksheet)
}
return connect.NewResponse(&v1pb.SearchWorksheetsResponse{
@@ -299,11 +284,7 @@ func (s *WorksheetService) UpdateWorksheet(
if worksheet == nil {
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("worksheet %q not found", request.Worksheet.Name))
}
- v1pbWorksheet, err := s.convertToAPIWorksheetMessage(ctx, worksheet)
- if err != nil {
- return nil, err
- }
-
+ v1pbWorksheet := convertToAPIWorksheetMessage(worksheet)
return connect.NewResponse(v1pbWorksheet), nil
}
@@ -526,20 +507,10 @@ func (s *WorksheetService) checkWorksheetPermission(
return ok, nil
}
-func (s *WorksheetService) convertToAPIWorksheetMessage(ctx context.Context, worksheet *store.WorkSheetMessage) (*v1pb.Worksheet, error) {
+func convertToAPIWorksheetMessage(worksheet *store.WorkSheetMessage) *v1pb.Worksheet {
databaseParent := ""
if worksheet.InstanceID != nil && worksheet.DatabaseName != nil {
- database, err := s.store.GetDatabase(ctx, &store.FindDatabaseMessage{
- ProjectID: &worksheet.ProjectID,
- InstanceID: worksheet.InstanceID,
- DatabaseName: worksheet.DatabaseName,
- })
- if err != nil {
- slog.Debug("failed to found database for worksheet", log.BBError(err), slog.Int("id", worksheet.UID), slog.String("instance", *worksheet.InstanceID), slog.String("database", *worksheet.DatabaseName))
- }
- if database != nil {
- databaseParent = common.FormatDatabase(database.InstanceID, database.DatabaseName)
- }
+ databaseParent = common.FormatDatabase(*worksheet.InstanceID, *worksheet.DatabaseName)
}
visibility := v1pb.Worksheet_VISIBILITY_UNSPECIFIED
@@ -553,27 +524,12 @@ func (s *WorksheetService) convertToAPIWorksheetMessage(ctx context.Context, wor
default:
// Keep VISIBILITY_UNSPECIFIED
}
-
- creator, err := s.store.GetUserByEmail(ctx, worksheet.Creator)
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to get creator: %v", err))
- }
-
- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{
- ResourceID: &worksheet.ProjectID,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to get project: %v", err))
- }
- if project == nil {
- return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("project with id %s not found", worksheet.ProjectID))
- }
return &v1pb.Worksheet{
Name: fmt.Sprintf("%s%d", common.WorksheetIDPrefix, worksheet.UID),
- Project: common.FormatProject(project.ResourceID),
+ Project: common.FormatProject(worksheet.ProjectID),
Database: databaseParent,
Title: worksheet.Title,
- Creator: fmt.Sprintf("users/%s", creator.Email),
+ Creator: fmt.Sprintf("users/%s", worksheet.Creator),
CreateTime: timestamppb.New(worksheet.CreatedAt),
UpdateTime: timestamppb.New(worksheet.UpdatedAt),
Content: []byte(worksheet.Statement),
@@ -581,7 +537,7 @@ func (s *WorksheetService) convertToAPIWorksheetMessage(ctx context.Context, wor
Visibility: visibility,
Starred: worksheet.Starred,
Folders: worksheet.Folders,
- }, nil
+ }
}
func convertToStoreWorksheetMessage(project *store.ProjectMessage, database *store.DatabaseMessage, creator string, worksheet *v1pb.Worksheet) (*store.WorkSheetMessage, error) {
diff --git a/backend/migrator/migration/3.14/0002##add_idx_instance_metadata_engine.sql b/backend/migrator/migration/3.14/0002##add_idx_instance_metadata_engine.sql
new file mode 100644
index 0000000000..ffb75a426e
--- /dev/null
+++ b/backend/migrator/migration/3.14/0002##add_idx_instance_metadata_engine.sql
@@ -0,0 +1 @@
+CREATE INDEX idx_instance_metadata_engine ON instance((metadata->>'engine'));
diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sql
index a2ce6e2c87..e8ae780cf9 100644
--- a/backend/migrator/migration/LATEST.sql
+++ b/backend/migrator/migration/LATEST.sql
@@ -130,6 +130,8 @@ CREATE TABLE instance (
CREATE UNIQUE INDEX idx_instance_unique_resource_id ON instance(resource_id);
+CREATE INDEX idx_instance_metadata_engine ON instance((metadata->>'engine'));
+
ALTER SEQUENCE instance_id_seq RESTART WITH 101;
-- db stores the databases for a particular instance
diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.go
index 34d644a46a..f518bec456 100644
--- a/backend/migrator/migrator_test.go
+++ b/backend/migrator/migrator_test.go
@@ -12,7 +12,7 @@ import (
func TestLatestVersion(t *testing.T) {
files, err := getSortedVersionedFiles()
require.NoError(t, err)
- require.Equal(t, semver.MustParse("3.14.1"), *files[len(files)-1].version)
+ require.Equal(t, semver.MustParse("3.14.2"), *files[len(files)-1].version)
}
func TestVersionUnique(t *testing.T) {
diff --git a/backend/runner/taskrun/database_migrate_executor.go b/backend/runner/taskrun/database_migrate_executor.go
index a836645124..807ffe1386 100644
--- a/backend/runner/taskrun/database_migrate_executor.go
+++ b/backend/runner/taskrun/database_migrate_executor.go
@@ -97,7 +97,7 @@ func (exec *DatabaseMigrateExecutor) runMigrationWithPriorBackup(ctx context.Con
})
// Check if we should skip backup or not.
- if common.EngineSupportPriorBackup(instance.Metadata.GetEngine()) {
+ if common.EngineSupportPriorBackup(database.Engine) {
var backupErr error
priorBackupDetail, backupErr = exec.backupData(ctx, driverCtx, statement, task.Payload, task, instance, database)
if backupErr != nil {
@@ -271,7 +271,7 @@ func (exec *DatabaseMigrateExecutor) backupData(
sourceDatabaseName := common.FormatDatabase(database.InstanceID, database.DatabaseName)
// Format: instances/{instance}/databases/{database}
- backupDBName := common.BackupDatabaseNameOfEngine(instance.Metadata.GetEngine())
+ backupDBName := common.BackupDatabaseNameOfEngine(database.Engine)
targetDatabaseName := common.FormatDatabase(database.InstanceID, backupDBName)
var backupDatabase *store.DatabaseMessage
var backupDriver db.Driver
@@ -281,7 +281,7 @@ func (exec *DatabaseMigrateExecutor) backupData(
return nil, errors.Wrap(err, "failed to parse backup database")
}
- if instance.Metadata.GetEngine() != storepb.Engine_POSTGRES {
+ if database.Engine != storepb.Engine_POSTGRES {
backupDatabase, err = exec.store.GetDatabase(ctx, &store.FindDatabaseMessage{InstanceID: &backupInstanceID, DatabaseName: &backupDatabaseName})
if err != nil {
return nil, errors.Wrap(err, "failed to get backup database")
@@ -315,7 +315,7 @@ func (exec *DatabaseMigrateExecutor) backupData(
IsCaseSensitive: store.IsObjectCaseSensitive(instance),
DatabaseName: database.DatabaseName,
}
- if instance.Metadata.GetEngine() == storepb.Engine_ORACLE {
+ if database.Engine == storepb.Engine_ORACLE {
oracleDriver, ok := driver.(*oracle.Driver)
if ok {
if version, err := oracleDriver.GetVersion(); err == nil {
@@ -329,12 +329,12 @@ func (exec *DatabaseMigrateExecutor) backupData(
}
prefix := "_" + time.Now().Format("20060102150405")
- statements, err := parserbase.TransformDMLToSelect(ctx, instance.Metadata.GetEngine(), tc, originStatement, database.DatabaseName, backupDatabaseName, prefix)
+ statements, err := parserbase.TransformDMLToSelect(ctx, database.Engine, tc, originStatement, database.DatabaseName, backupDatabaseName, prefix)
if err != nil {
return nil, errors.Wrap(err, "failed to transform DML to select")
}
- prependStatements, err := getPrependStatements(instance.Metadata.GetEngine(), originStatement)
+ prependStatements, err := getPrependStatements(database.Engine, originStatement)
if err != nil {
return nil, errors.Wrap(err, "failed to get prepend statements")
}
@@ -396,7 +396,7 @@ func (exec *DatabaseMigrateExecutor) backupData(
StartPosition: statement.StartPosition,
EndPosition: statement.EndPosition,
}
- if instance.Metadata.GetEngine() == storepb.Engine_POSTGRES {
+ if database.Engine == storepb.Engine_POSTGRES {
item.TargetTable = &storepb.PriorBackupDetail_Item_Table{
Database: sourceDatabaseName,
// postgres uses schema as the backup database name currently.
@@ -407,7 +407,7 @@ func (exec *DatabaseMigrateExecutor) backupData(
priorBackupDetail.Items = append(priorBackupDetail.Items, item)
}
- if instance.Metadata.GetEngine() != storepb.Engine_POSTGRES {
+ if database.Engine != storepb.Engine_POSTGRES {
if err := exec.schemaSyncer.SyncDatabaseSchema(ctx, backupDatabase); err != nil {
slog.Error("failed to sync backup database schema",
slog.String("database", targetDatabaseName),
diff --git a/backend/runner/taskrun/executor.go b/backend/runner/taskrun/executor.go
index 74fc69c1cb..d243bcb62d 100644
--- a/backend/runner/taskrun/executor.go
+++ b/backend/runner/taskrun/executor.go
@@ -62,9 +62,8 @@ type migrateContext struct {
profile *config.Profile
dbFactory *dbfactory.DBFactory
- instance *store.InstanceMessage
- database *store.DatabaseMessage
- // nullable if type=baseline
+ instance *store.InstanceMessage
+ database *store.DatabaseMessage
sheetSha256 string
task *store.TaskMessage
@@ -141,7 +140,7 @@ func runMigrationWithFunc(
// Pre-compute whether schema dump is needed.
// Skip dump for pure DML statements (INSERT, UPDATE, DELETE) as they don't change schema.
- mc.needDump = computeNeedDump(task.Type, mc.instance.Metadata.GetEngine(), statement)
+ mc.needDump = computeNeedDump(task.Type, mc.database.Engine, statement)
skipped, err := doMigrationWithFunc(ctx, driverCtx, store, stateCfg, profile, statement, mc, execFunc)
if err != nil {
@@ -240,7 +239,7 @@ func doMigrationWithFunc(
statementRecord, _ := common.TruncateString(statement, common.MaxSheetSize)
slog.Debug("Start migration...",
- slog.String("instance", instance.ResourceID),
+ slog.String("instance", database.InstanceID),
slog.String("database", database.DatabaseName),
slog.String("type", string(mc.task.Type.String())),
slog.String("statement", statementRecord),
diff --git a/backend/store/database.go b/backend/store/database.go
index bbb091869f..1ecfa8d704 100644
--- a/backend/store/database.go
+++ b/backend/store/database.go
@@ -30,6 +30,7 @@ type DatabaseMessage struct {
Deleted bool
Metadata *storepb.DatabaseMetadata
+ Engine storepb.Engine
}
func (d *DatabaseMessage) String() string {
@@ -144,7 +145,7 @@ func (s *Store) ListDatabases(ctx context.Context, find *FindDatabaseMessage) ([
where.And("db.name = ?", *v)
}
if v := find.Engine; v != nil {
- where.And("instance.metadata->>'engine' = ?", *v)
+ where.And("instance.metadata->>'engine' = ?", v.String())
}
if !find.ShowDeleted {
where.And("instance.deleted = ?", false)
@@ -162,7 +163,8 @@ func (s *Store) ListDatabases(ctx context.Context, find *FindDatabaseMessage) ([
db.instance,
db.name,
db.deleted,
- db.metadata
+ db.metadata,
+ instance.metadata->>'engine'
FROM ?
WHERE ?
ORDER BY db.project, db.instance, db.name
@@ -189,7 +191,7 @@ func (s *Store) ListDatabases(ctx context.Context, find *FindDatabaseMessage) ([
for rows.Next() {
databaseMessage := &DatabaseMessage{}
var metadataString string
- var effectiveEnvironment, environment sql.NullString
+ var effectiveEnvironment, environment, engine sql.NullString
if err := rows.Scan(
&databaseMessage.ProjectID,
&effectiveEnvironment,
@@ -198,6 +200,7 @@ func (s *Store) ListDatabases(ctx context.Context, find *FindDatabaseMessage) ([
&databaseMessage.DatabaseName,
&databaseMessage.Deleted,
&metadataString,
+ &engine,
); err != nil {
return nil, err
}
@@ -207,6 +210,11 @@ func (s *Store) ListDatabases(ctx context.Context, find *FindDatabaseMessage) ([
if environment.Valid {
databaseMessage.EnvironmentID = &environment.String
}
+ if engine.Valid {
+ if v, ok := storepb.Engine_value[engine.String]; ok {
+ databaseMessage.Engine = storepb.Engine(v)
+ }
+ }
var metadata storepb.DatabaseMetadata
if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(metadataString), &metadata); err != nil {
@@ -586,7 +594,7 @@ func GetListDatabaseFilter(filter string) (*qb.Query, error) {
return nil, errors.Errorf("invalid engine filter %q", value)
}
engine := storepb.Engine(engineValue)
- return qb.Q().Space("instance.metadata->>'engine' = ?", engine), nil
+ return qb.Q().Space("instance.metadata->>'engine' = ?", engine.String()), nil
case "name":
return qb.Q().Space("db.name = ?", value), nil
case "drifted":