Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

test(backend): keep one subtest's sync from deleting another's database (#21759)

Danny Xua38fbf43f398
diff --git a/backend/tests/database.go b/backend/tests/database.go
index 696e66d5e6..eb50880bbf 100644
--- a/backend/tests/database.go
+++ b/backend/tests/database.go
@@ -9,6 +9,7 @@ import (
"connectrpc.com/connect"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
+ "github.com/pkg/errors"
"google.golang.org/protobuf/types/known/fieldmaskpb"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
@@ -37,7 +38,7 @@ func (ctl *controller) createDatabase(ctx context.Context, project *v1pb.Project
return err
}
}
- if err := ctl.registerDatabase(ctx, instance, databaseName); err != nil {
+ if err := ctl.waitDatabaseRegistered(ctx, instance, databaseName); err != nil {
return err
}
@@ -151,49 +152,84 @@ func (ctl *controller) allowJustInTimeAccess(ctx context.Context, project *v1pb.
return err
}
-// instanceLocks serializes registerDatabase per instance: parallel subtests
-// share instances, and a sync that reads the list before another subtest's
-// update marks that subtest's database deleted.
+// instanceSyncLists holds, per instance, the sync list registerDatabase has
+// written and the lock that serializes those writes. The list is rebuilt from
+// this record rather than from a read of the instance: the server serves
+// instances from a cache, so a read can miss another subtest's update, and
+// writing the list back without that subtest's database makes the next sync
+// mark it deleted under the test still using it.
var (
- instanceLocksMu sync.Mutex
- instanceLocks = map[string]*sync.Mutex{}
+ instanceSyncListsMu sync.Mutex
+ instanceSyncLists = map[string]*instanceSyncList{}
)
-func instanceLock(name string) *sync.Mutex {
- instanceLocksMu.Lock()
- defer instanceLocksMu.Unlock()
- if instanceLocks[name] == nil {
- instanceLocks[name] = &sync.Mutex{}
+type instanceSyncList struct {
+ mu sync.Mutex
+ databases []string
+}
+
+func instanceSyncListOf(instance *v1pb.Instance) *instanceSyncList {
+ instanceSyncListsMu.Lock()
+ defer instanceSyncListsMu.Unlock()
+ if instanceSyncLists[instance.Name] == nil {
+ instanceSyncLists[instance.Name] = &instanceSyncList{databases: slices.Clone(instance.GetSyncDatabases().GetDatabases())}
}
- return instanceLocks[name]
+ return instanceSyncLists[instance.Name]
}
// registerDatabase names a database in the instance's sync list, which an
// instance on the shared target carries so it never imports another test's
// databases, and syncs the instance. An instance with a server to itself keeps
-// no list and only syncs.
+// no list and only syncs. The database need not exist yet: the list may name
+// databases the target does not hold, and the sync passes over them.
func (ctl *controller) registerDatabase(ctx context.Context, instance *v1pb.Instance, databaseName string) error {
- mu := instanceLock(instance.Name)
- mu.Lock()
- defer mu.Unlock()
+ syncList := instanceSyncListOf(instance)
+ syncList.mu.Lock()
+ defer syncList.mu.Unlock()
+ if !slices.Contains(syncList.databases, databaseName) {
+ syncList.databases = append(syncList.databases, databaseName)
+ }
if instance.GetSyncDatabases() != nil {
- resp, err := ctl.instanceServiceClient.GetInstance(ctx, connect.NewRequest(&v1pb.GetInstanceRequest{Name: instance.Name}))
- if err != nil {
+ if _, err := ctl.instanceServiceClient.UpdateInstance(ctx, connect.NewRequest(&v1pb.UpdateInstanceRequest{
+ Instance: &v1pb.Instance{
+ Name: instance.Name,
+ SyncDatabases: &v1pb.SyncDatabases{Databases: slices.Clone(syncList.databases)},
+ },
+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"sync_databases"}},
+ })); err != nil {
return err
}
- if databases := resp.Msg.GetSyncDatabases().GetDatabases(); !slices.Contains(databases, databaseName) {
- if _, err := ctl.instanceServiceClient.UpdateInstance(ctx, connect.NewRequest(&v1pb.UpdateInstanceRequest{
- Instance: &v1pb.Instance{
- Name: instance.Name,
- SyncDatabases: &v1pb.SyncDatabases{Databases: append(databases, databaseName)},
- },
- UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"sync_databases"}},
- })); err != nil {
- return err
- }
- }
}
_, err := ctl.instanceServiceClient.SyncInstance(ctx, connect.NewRequest(&v1pb.SyncInstanceRequest{Name: instance.Name}))
return err
}
+
+// instanceSyncAttempts bounds the syncs waitDatabaseRegistered spends on one
+// database.
+const instanceSyncAttempts = 3
+
+// waitDatabaseRegistered registers a database the target already holds and
+// returns once the instance sync has given it an active resource, which the
+// caller needs before it can address the database. One sync can miss it: the
+// sync reads the list through the server's instance cache, which a concurrent
+// read can refill with a copy taken before this database joined it.
+func (ctl *controller) waitDatabaseRegistered(ctx context.Context, instance *v1pb.Instance, databaseName string) error {
+ name := fmt.Sprintf("%s/databases/%s", instance.Name, databaseName)
+ var last error
+ for range instanceSyncAttempts {
+ if err := ctl.registerDatabase(ctx, instance, databaseName); err != nil {
+ return err
+ }
+ resp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{Name: name}))
+ switch {
+ case err != nil:
+ last = err
+ case resp.Msg.State != v1pb.State_ACTIVE:
+ last = errors.Errorf("database %s is %s", name, resp.Msg.State)
+ default:
+ return nil
+ }
+ }
+ return errors.Wrapf(last, "database %s is not registered after %d instance syncs", name, instanceSyncAttempts)
+}
diff --git a/backend/tests/database_test.go b/backend/tests/database_test.go
new file mode 100644
index 0000000000..1394b19ac2
--- /dev/null
+++ b/backend/tests/database_test.go
@@ -0,0 +1,61 @@
+package tests
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "connectrpc.com/connect"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/types/known/fieldmaskpb"
+
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+// TestCreateDatabaseKeepsEarlierDatabasesRegistered pins what tests with
+// parallel subtests on one instance depend on: the instance sync marks deleted
+// every database the instance's sync list leaves out, so createDatabase writes
+// that list from every database it has registered rather than from a read of
+// the instance, which the server can serve from a cache filled before another
+// subtest's update. The out-of-band write below stands in for such a read.
+func TestCreateDatabaseKeepsEarlierDatabasesRegistered(t *testing.T) {
+ t.Parallel()
+ a := require.New(t)
+ ctx := context.Background()
+ ctl, ctx := startProject(ctx, t)
+
+ pgContainer := sharedPgTarget(t)
+ instanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{
+ InstanceId: generateRandomString("instance"),
+ Instance: &v1pb.Instance{
+ SyncDatabases: &v1pb.SyncDatabases{},
+ Title: "sync list instance",
+ Engine: v1pb.Engine_POSTGRES,
+ Environment: new("environments/prod"),
+ Activation: true,
+ DataSources: []*v1pb.DataSource{pgContainer.adminDataSource()},
+ },
+ }))
+ a.NoError(err)
+ instance := instanceResp.Msg
+
+ first, second := uniqueDB("sync_list_first"), uniqueDB("sync_list_second")
+ a.NoError(ctl.createDatabase(ctx, ctl.project, instance, nil, first, "postgres"))
+
+ _, err = ctl.instanceServiceClient.UpdateInstance(ctx, connect.NewRequest(&v1pb.UpdateInstanceRequest{
+ Instance: &v1pb.Instance{
+ Name: instance.Name,
+ SyncDatabases: &v1pb.SyncDatabases{},
+ },
+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"sync_databases"}},
+ }))
+ a.NoError(err)
+
+ a.NoError(ctl.createDatabase(ctx, ctl.project, instance, nil, second, "postgres"))
+
+ firstResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{
+ Name: fmt.Sprintf("%s/databases/%s", instance.Name, first),
+ }))
+ a.NoError(err)
+ a.Equal(v1pb.State_ACTIVE, firstResp.Msg.State, "registering a database must leave the ones registered before it")
+}
diff --git a/backend/tests/sql_stop_on_error_test.go b/backend/tests/sql_stop_on_error_test.go
index 6171129642..c6c13f8b29 100644
--- a/backend/tests/sql_stop_on_error_test.go
+++ b/backend/tests/sql_stop_on_error_test.go
@@ -88,7 +88,7 @@ func TestSQLQueryStopOnError(t *testing.T) {
instance := pgInstance
databaseOwner := "postgres"
- err = ctl.createDatabase(ctx, ctl.project, instance, nil, tt.databaseName, databaseOwner)
+ err := ctl.createDatabase(ctx, ctl.project, instance, nil, tt.databaseName, databaseOwner)
a.NoError(err)
databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{
@@ -206,7 +206,7 @@ func TestSQLAdminExecuteStopOnError(t *testing.T) {
instance := pgInstance
databaseOwner := "postgres"
- err = ctl.createDatabase(ctx, ctl.project, instance, nil, tt.databaseName, databaseOwner)
+ err := ctl.createDatabase(ctx, ctl.project, instance, nil, tt.databaseName, databaseOwner)
a.NoError(err)
databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{