Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
test(backend): keep one subtest's sync from deleting another's database (#21759)
Danny Xua38fbf43f398
diff --git a/backend/tests/database.go b/backend/tests/database.goindex 696e66d5e6..eb50880bbf 100644--- a/backend/tests/database.go+++ b/backend/tests/database.go@@ -9,6 +9,7 @@ import ("connectrpc.com/connect""github.com/google/uuid""github.com/jackc/pgx/v5"+ "github.com/pkg/errors""google.golang.org/protobuf/types/known/fieldmaskpb"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"@@ -37,7 +38,7 @@ func (ctl *controller) createDatabase(ctx context.Context, project *v1pb.Projectreturn err}}- if err := ctl.registerDatabase(ctx, instance, databaseName); err != nil {+ if err := ctl.waitDatabaseRegistered(ctx, instance, databaseName); err != nil {return err}@@ -151,49 +152,84 @@ func (ctl *controller) allowJustInTimeAccess(ctx context.Context, project *v1pb.return err}-// instanceLocks serializes registerDatabase per instance: parallel subtests-// share instances, and a sync that reads the list before another subtest's-// update marks that subtest's database deleted.+// instanceSyncLists holds, per instance, the sync list registerDatabase has+// written and the lock that serializes those writes. The list is rebuilt from+// this record rather than from a read of the instance: the server serves+// instances from a cache, so a read can miss another subtest's update, and+// writing the list back without that subtest's database makes the next sync+// mark it deleted under the test still using it.var (- instanceLocksMu sync.Mutex- instanceLocks = map[string]*sync.Mutex{}+ instanceSyncListsMu sync.Mutex+ instanceSyncLists = map[string]*instanceSyncList{})-func instanceLock(name string) *sync.Mutex {- instanceLocksMu.Lock()- defer instanceLocksMu.Unlock()- if instanceLocks[name] == nil {- instanceLocks[name] = &sync.Mutex{}+type instanceSyncList struct {+ mu sync.Mutex+ databases []string+}++func instanceSyncListOf(instance *v1pb.Instance) *instanceSyncList {+ instanceSyncListsMu.Lock()+ defer instanceSyncListsMu.Unlock()+ if instanceSyncLists[instance.Name] == nil {+ instanceSyncLists[instance.Name] = &instanceSyncList{databases: slices.Clone(instance.GetSyncDatabases().GetDatabases())}}- return instanceLocks[name]+ return instanceSyncLists[instance.Name]}// registerDatabase names a database in the instance's sync list, which an// instance on the shared target carries so it never imports another test's// databases, and syncs the instance. An instance with a server to itself keeps-// no list and only syncs.+// no list and only syncs. The database need not exist yet: the list may name+// databases the target does not hold, and the sync passes over them.func (ctl *controller) registerDatabase(ctx context.Context, instance *v1pb.Instance, databaseName string) error {- mu := instanceLock(instance.Name)- mu.Lock()- defer mu.Unlock()+ syncList := instanceSyncListOf(instance)+ syncList.mu.Lock()+ defer syncList.mu.Unlock()+ if !slices.Contains(syncList.databases, databaseName) {+ syncList.databases = append(syncList.databases, databaseName)+ }if instance.GetSyncDatabases() != nil {- resp, err := ctl.instanceServiceClient.GetInstance(ctx, connect.NewRequest(&v1pb.GetInstanceRequest{Name: instance.Name}))- if err != nil {+ if _, err := ctl.instanceServiceClient.UpdateInstance(ctx, connect.NewRequest(&v1pb.UpdateInstanceRequest{+ Instance: &v1pb.Instance{+ Name: instance.Name,+ SyncDatabases: &v1pb.SyncDatabases{Databases: slices.Clone(syncList.databases)},+ },+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"sync_databases"}},+ })); err != nil {return err}- if databases := resp.Msg.GetSyncDatabases().GetDatabases(); !slices.Contains(databases, databaseName) {- if _, err := ctl.instanceServiceClient.UpdateInstance(ctx, connect.NewRequest(&v1pb.UpdateInstanceRequest{- Instance: &v1pb.Instance{- Name: instance.Name,- SyncDatabases: &v1pb.SyncDatabases{Databases: append(databases, databaseName)},- },- UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"sync_databases"}},- })); err != nil {- return err- }- }}_, err := ctl.instanceServiceClient.SyncInstance(ctx, connect.NewRequest(&v1pb.SyncInstanceRequest{Name: instance.Name}))return err}++// instanceSyncAttempts bounds the syncs waitDatabaseRegistered spends on one+// database.+const instanceSyncAttempts = 3++// waitDatabaseRegistered registers a database the target already holds and+// returns once the instance sync has given it an active resource, which the+// caller needs before it can address the database. One sync can miss it: the+// sync reads the list through the server's instance cache, which a concurrent+// read can refill with a copy taken before this database joined it.+func (ctl *controller) waitDatabaseRegistered(ctx context.Context, instance *v1pb.Instance, databaseName string) error {+ name := fmt.Sprintf("%s/databases/%s", instance.Name, databaseName)+ var last error+ for range instanceSyncAttempts {+ if err := ctl.registerDatabase(ctx, instance, databaseName); err != nil {+ return err+ }+ resp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{Name: name}))+ switch {+ case err != nil:+ last = err+ case resp.Msg.State != v1pb.State_ACTIVE:+ last = errors.Errorf("database %s is %s", name, resp.Msg.State)+ default:+ return nil+ }+ }+ return errors.Wrapf(last, "database %s is not registered after %d instance syncs", name, instanceSyncAttempts)+}diff --git a/backend/tests/database_test.go b/backend/tests/database_test.gonew file mode 100644index 0000000000..1394b19ac2--- /dev/null+++ b/backend/tests/database_test.go@@ -0,0 +1,61 @@+package tests++import (+ "context"+ "fmt"+ "testing"++ "connectrpc.com/connect"+ "github.com/stretchr/testify/require"+ "google.golang.org/protobuf/types/known/fieldmaskpb"++ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// TestCreateDatabaseKeepsEarlierDatabasesRegistered pins what tests with+// parallel subtests on one instance depend on: the instance sync marks deleted+// every database the instance's sync list leaves out, so createDatabase writes+// that list from every database it has registered rather than from a read of+// the instance, which the server can serve from a cache filled before another+// subtest's update. The out-of-band write below stands in for such a read.+func TestCreateDatabaseKeepsEarlierDatabasesRegistered(t *testing.T) {+ t.Parallel()+ a := require.New(t)+ ctx := context.Background()+ ctl, ctx := startProject(ctx, t)++ pgContainer := sharedPgTarget(t)+ instanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{+ InstanceId: generateRandomString("instance"),+ Instance: &v1pb.Instance{+ SyncDatabases: &v1pb.SyncDatabases{},+ Title: "sync list instance",+ Engine: v1pb.Engine_POSTGRES,+ Environment: new("environments/prod"),+ Activation: true,+ DataSources: []*v1pb.DataSource{pgContainer.adminDataSource()},+ },+ }))+ a.NoError(err)+ instance := instanceResp.Msg++ first, second := uniqueDB("sync_list_first"), uniqueDB("sync_list_second")+ a.NoError(ctl.createDatabase(ctx, ctl.project, instance, nil, first, "postgres"))++ _, err = ctl.instanceServiceClient.UpdateInstance(ctx, connect.NewRequest(&v1pb.UpdateInstanceRequest{+ Instance: &v1pb.Instance{+ Name: instance.Name,+ SyncDatabases: &v1pb.SyncDatabases{},+ },+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"sync_databases"}},+ }))+ a.NoError(err)++ a.NoError(ctl.createDatabase(ctx, ctl.project, instance, nil, second, "postgres"))++ firstResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{+ Name: fmt.Sprintf("%s/databases/%s", instance.Name, first),+ }))+ a.NoError(err)+ a.Equal(v1pb.State_ACTIVE, firstResp.Msg.State, "registering a database must leave the ones registered before it")+}diff --git a/backend/tests/sql_stop_on_error_test.go b/backend/tests/sql_stop_on_error_test.goindex 6171129642..c6c13f8b29 100644--- a/backend/tests/sql_stop_on_error_test.go+++ b/backend/tests/sql_stop_on_error_test.go@@ -88,7 +88,7 @@ func TestSQLQueryStopOnError(t *testing.T) {instance := pgInstancedatabaseOwner := "postgres"- err = ctl.createDatabase(ctx, ctl.project, instance, nil, tt.databaseName, databaseOwner)+ err := ctl.createDatabase(ctx, ctl.project, instance, nil, tt.databaseName, databaseOwner)a.NoError(err)databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{@@ -206,7 +206,7 @@ func TestSQLAdminExecuteStopOnError(t *testing.T) {instance := pgInstancedatabaseOwner := "postgres"- err = ctl.createDatabase(ctx, ctl.project, instance, nil, tt.databaseName, databaseOwner)+ err := ctl.createDatabase(ctx, ctl.project, instance, nil, tt.databaseName, databaseOwner)a.NoError(err)databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{