Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
perf(tests): direct database fixtures, read-first rollout waits, leaner webhook test (#21434)
Danny Xu2400a4cd95fe
diff --git a/backend/runner/taskrun/scheduler.go b/backend/runner/taskrun/scheduler.goindex f9b0ed1146..15295b7616 100644--- a/backend/runner/taskrun/scheduler.go+++ b/backend/runner/taskrun/scheduler.go@@ -198,6 +198,23 @@ func (s *Scheduler) runTaskCompletionListener(ctx context.Context) {}}+// planTasksComplete reports whether every task has reached a state that counts+// as finishing the pipeline: a task run that is DONE or SKIPPED, or a task the+// user skipped. FAILED and CANCELED do not count, so a pipeline holding one is+// not complete until it is retried into DONE or skipped.+func planTasksComplete(tasks []*store.TaskMessage) bool {+ for _, task := range tasks {+ switch {+ case task.LatestTaskRunStatus == storepb.TaskRun_DONE,+ task.LatestTaskRunStatus == storepb.TaskRun_SKIPPED,+ task.Payload.GetSkipped():+ default:+ return false+ }+ }+ return true+}+// checkPlanCompletion checks if all tasks in a plan are complete and successful.// If so, sends PIPELINE_COMPLETED webhook and auto-resolves issues for deferred rollout plans.// Deferred rollout plans (createDatabaseConfig) auto-resolve when tasks complete.@@ -217,20 +234,8 @@ func (s *Scheduler) checkPlanCompletion(ctx context.Context, ref bus.PlanRef) {return}- // Check if all tasks are complete (DONE or SKIPPED)- for _, task := range tasks {- status := task.LatestTaskRunStatus-- // Only DONE and SKIPPED are considered complete- // FAILED and CANCELED are not complete states- isComplete := status == storepb.TaskRun_DONE ||- status == storepb.TaskRun_SKIPPED ||- task.Payload.GetSkipped()-- if !isComplete {- // Not all tasks complete - no webhook- return- }+ if !planTasksComplete(tasks) {+ return}project, err := s.store.GetProjectByResourceID(ctx, plan.ProjectID)diff --git a/backend/runner/taskrun/scheduler_test.go b/backend/runner/taskrun/scheduler_test.goindex bbc259ea1c..5fff01cf36 100644--- a/backend/runner/taskrun/scheduler_test.go+++ b/backend/runner/taskrun/scheduler_test.go@@ -101,6 +101,40 @@ func TestCompletionWebhookEnvironmentUsesLastEnvironmentOrder(t *testing.T) {require.Equal(t, "prod", completionWebhookEnvironment(tasks, environmentOrderMap))}+// A retried task reaches DONE, so "all done" is the retry case too.+func TestPlanTasksComplete(t *testing.T) {+ done := &store.TaskMessage{LatestTaskRunStatus: storepb.TaskRun_DONE}+ skipped := &store.TaskMessage{LatestTaskRunStatus: storepb.TaskRun_SKIPPED}+ userSkipped := &store.TaskMessage{Payload: &storepb.Task{Skipped: true}}+ failed := &store.TaskMessage{LatestTaskRunStatus: storepb.TaskRun_FAILED}+ canceled := &store.TaskMessage{LatestTaskRunStatus: storepb.TaskRun_CANCELED}+ running := &store.TaskMessage{LatestTaskRunStatus: storepb.TaskRun_RUNNING}+ pending := &store.TaskMessage{LatestTaskRunStatus: storepb.TaskRun_PENDING}++ testCases := []struct {+ name string+ tasks []*store.TaskMessage+ want bool+ }{+ {"all done", []*store.TaskMessage{done, done}, true},+ {"done and skipped", []*store.TaskMessage{done, skipped}, true},+ {"all skipped", []*store.TaskMessage{skipped, skipped}, true},+ {"a failed task the user skipped", []*store.TaskMessage{done, userSkipped}, true},+ {"every failed task skipped", []*store.TaskMessage{userSkipped, userSkipped}, true},+ {"mixed recovery", []*store.TaskMessage{done, skipped, done, userSkipped}, true},+ {"no tasks", nil, true},+ {"one still failed", []*store.TaskMessage{done, failed}, false},+ {"one canceled", []*store.TaskMessage{done, canceled}, false},+ {"one still running", []*store.TaskMessage{done, running}, false},+ {"one still pending", []*store.TaskMessage{done, pending}, false},+ }+ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ require.Equal(t, tc.want, planTasksComplete(tc.tasks))+ })+ }+}+func TestSchedulePendingTaskRunsSkipsArchivedProject(t *testing.T) {ctx := context.Background()s := setupRolloutCreatorStore(ctx, t)diff --git a/backend/store/plan_webhook_delivery_test.go b/backend/store/plan_webhook_delivery_test.gonew file mode 100644index 0000000000..8410bc3ec0--- /dev/null+++ b/backend/store/plan_webhook_delivery_test.go@@ -0,0 +1,68 @@+package store_test++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/store"+)++// TestPlanWebhookDeliveryClaims pins the state machine behind PIPELINE_COMPLETED+// and PIPELINE_FAILED: one row per plan, so the first claim wins and the second+// event of either kind stays silent until a retry resets the row.+func TestPlanWebhookDeliveryClaims(t *testing.T) {+ t.Parallel()+ ctx := context.Background()+ db, stores, _ := testcontainer.NewMetadataDB(t)+ _, err := db.ExecContext(ctx, `+ INSERT INTO workspace (resource_id) VALUES ('default');+ INSERT INTO principal (name, email, password_hash) VALUES ('c', '[email protected]', 'x');+ INSERT INTO project (resource_id, workspace, name) VALUES ('p1', 'default', 'P1'), ('p2', 'default', 'P2');+ `)+ require.NoError(t, err)++ newPlan := func(projectID string) int64 {+ plan, err := stores.CreatePlan(ctx, &store.PlanMessage{+ ProjectID: projectID, Name: "plan", Config: &storepb.PlanConfig{},+ }, "[email protected]")+ require.NoError(t, err)+ return plan.UID+ }+ planA, planB := newPlan("p1"), newPlan("p2")++ claimed, err := stores.ClaimPipelineCompletionNotification(ctx, "p1", planA)+ require.NoError(t, err)+ require.True(t, claimed, "the first completion claims the plan")++ claimed, err = stores.ClaimPipelineCompletionNotification(ctx, "p1", planA)+ require.NoError(t, err)+ require.False(t, claimed, "a second completion for the same plan sends nothing")++ // Both events share the row, so a completed plan cannot then report failure.+ claimed, err = stores.ClaimPipelineFailureNotification(ctx, "p1", planA)+ require.NoError(t, err)+ require.False(t, claimed)++ // A retry resets the row, which is what re-arms both events.+ require.NoError(t, stores.ResetPlanWebhookDelivery(ctx, "p1", planA))+ claimed, err = stores.ClaimPipelineFailureNotification(ctx, "p1", planA)+ require.NoError(t, err)+ require.True(t, claimed, "the reset re-arms the plan")++ // Plan UIDs restart per project, so a neighbor's claim must not be read as+ // this plan's.+ require.Equal(t, planA, planB, "plan UIDs must collide across projects for this to prove anything")+ claimed, err = stores.ClaimPipelineCompletionNotification(ctx, "p2", planB)+ require.NoError(t, err)+ require.True(t, claimed, "project p2 claims its own plan")++ var rows int+ require.NoError(t, db.QueryRowContext(ctx,+ `SELECT COUNT(*) FROM plan_webhook_delivery WHERE project = $1 AND plan_id = $2 AND event_type = 'PIPELINE_FAILED'`,+ "p1", planA).Scan(&rows))+ require.Equal(t, 1, rows)+}diff --git a/backend/tests/action_test.go b/backend/tests/action_test.goindex f3de4e168f..1a3c986782 100644--- a/backend/tests/action_test.go+++ b/backend/tests/action_test.go@@ -557,7 +557,7 @@ func (ctl *controller) createTestMySQLDatabase(ctx context.Context, t *testing.T// Create databasedbName := generateRandomString("db")[:8]- err = ctl.createDatabase(ctx, ctl.project, instanceResp.Msg, nil, dbName, "")+ err = ctl.createDatabaseByRollout(ctx, ctl.project, instanceResp.Msg, nil, dbName)a.NoError(err)// Get databasediff --git a/backend/tests/database.go b/backend/tests/database.goindex a2167bbb14..aa428f36e1 100644--- a/backend/tests/database.go+++ b/backend/tests/database.go@@ -4,64 +4,76 @@ import ("context""fmt""slices"+ "sync""connectrpc.com/connect""github.com/google/uuid"+ "github.com/jackc/pgx/v5""google.golang.org/protobuf/types/known/fieldmaskpb"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1")-// allowDatabaseSync names one more database an instance may sync. The syncer-// imports only what the list names, so a database about to be created has to be-// added first. An instance with a server to itself carries no list and skips this.-func (ctl *controller) allowDatabaseSync(ctx context.Context, instance *v1pb.Instance, databaseName string) error {- if instance.GetSyncDatabases() == nil {- return nil- }- resp, err := ctl.instanceServiceClient.GetInstance(ctx, connect.NewRequest(&v1pb.GetInstanceRequest{Name: instance.Name}))+// createDatabase gives a test a database on a Postgres instance and registers+// it in the project, as the create-database task leaves it: schema synced, and+// the environment set when one is given. It skips the rollout, which costs a+// plan, an issue, a task and a PIPELINE_COMPLETED webhook per database;+// TestDatabaseEnvironment keeps that path covered, through+// createDatabaseByRollout.+func (ctl *controller) createDatabase(ctx context.Context, project *v1pb.Project, instance *v1pb.Instance, environment *v1pb.EnvironmentSetting_Environment, databaseName string, owner string) error {+ pg, err := pgContainerOf(instance)if err != nil {return err}- databases := resp.Msg.GetSyncDatabases().GetDatabases()- if slices.Contains(databases, databaseName) {- return nil+ if err := ctl.allowLastPlanEditorApproval(ctx, project); err != nil {+ return err}- _, err = ctl.instanceServiceClient.UpdateInstance(ctx, connect.NewRequest(&v1pb.UpdateInstanceRequest{- Instance: &v1pb.Instance{- Name: instance.Name,- SyncDatabases: &v1pb.SyncDatabases{Databases: append(databases, databaseName)},- },- UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"sync_databases"}},- }))+ name := pgx.Identifier{databaseName}.Sanitize()+ if _, err := pg.GetDB().ExecContext(ctx, "CREATE DATABASE "+name); err != nil {+ return err+ }+ if owner != "" {+ if _, err := pg.GetDB().ExecContext(ctx, "ALTER DATABASE "+name+" OWNER TO "+pgx.Identifier{owner}.Sanitize()); err != nil {+ return err+ }+ }+ if err := ctl.registerDatabase(ctx, instance, databaseName); err != nil {+ return err+ }++ database := &v1pb.Database{+ Name: fmt.Sprintf("%s/databases/%s", instance.Name, databaseName),+ Project: project.Name,+ }+ paths := []string{"project"}+ if environment != nil {+ database.Environment = &environment.Name+ paths = append(paths, "environment")+ }+ if _, err := ctl.databaseServiceClient.UpdateDatabase(ctx, connect.NewRequest(&v1pb.UpdateDatabaseRequest{+ Database: database,+ UpdateMask: &fieldmaskpb.FieldMask{Paths: paths},+ })); err != nil {+ return err+ }+ _, err = ctl.databaseServiceClient.SyncDatabase(ctx, connect.NewRequest(&v1pb.SyncDatabaseRequest{Name: database.Name}))return err}-func (ctl *controller) createDatabase(ctx context.Context, project *v1pb.Project, instance *v1pb.Instance, environment *v1pb.EnvironmentSetting_Environment, databaseName string, owner string) error {- if err := ctl.allowDatabaseSync(ctx, instance, databaseName); err != nil {+// createDatabaseByRollout creates a database through the create-database plan,+// issue and rollout, for the engines createDatabase does not reach and the test+// that covers the workflow itself.+func (ctl *controller) createDatabaseByRollout(ctx context.Context, project *v1pb.Project, instance *v1pb.Instance, environment *v1pb.EnvironmentSetting_Environment, databaseName string) error {+ if err := ctl.registerDatabase(ctx, instance, databaseName); err != nil {return err}-- // Database provisioning is test setup, and its plan is approved by the same- // test actor. Opt into that legacy behavior explicitly; production projects- // keep the restrictive default when the setting is absent.- if _, err := ctl.projectServiceClient.UpdateProject(ctx, connect.NewRequest(&v1pb.UpdateProjectRequest{- Project: &v1pb.Project{- Name: project.Name,- AllowLastPlanEditorApproval: true,- },- UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"allow_last_plan_editor_approval"}},- })); err != nil {+ if err := ctl.allowLastPlanEditorApproval(ctx, project); err != nil {return err}- characterSet, collation := "utf8mb4", "utf8mb4_general_ci"+ characterSet, collation, owner := "utf8mb4", "utf8mb4_general_ci", ""if instance.Engine == v1pb.Engine_POSTGRES {- characterSet = "UTF8"- collation = "en_US.UTF-8"- if owner == "" {- owner = "postgres"- }+ characterSet, collation, owner = "UTF8", "en_US.UTF-8", "postgres"}environmentName := ""if environment != nil {@@ -110,3 +122,65 @@ func (ctl *controller) createDatabase(ctx context.Context, project *v1pb.Projectreturn ctl.waitRollout(ctx, issueResp.Msg.Name, rolloutResp.Msg.Name)}++// allowLastPlanEditorApproval opts the project into approving a plan its own+// editor wrote. Provisioning is test setup approved by the same test actor, and+// tests go on to approve their own plans; production projects keep the+// restrictive default.+func (ctl *controller) allowLastPlanEditorApproval(ctx context.Context, project *v1pb.Project) error {+ _, err := ctl.projectServiceClient.UpdateProject(ctx, connect.NewRequest(&v1pb.UpdateProjectRequest{+ Project: &v1pb.Project{+ Name: project.Name,+ AllowLastPlanEditorApproval: true,+ },+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"allow_last_plan_editor_approval"}},+ }))+ return err+}++// instanceLocks serializes registerDatabase per instance: parallel subtests+// share instances, and a sync that reads the list before another subtest's+// update marks that subtest's database deleted.+var (+ instanceLocksMu sync.Mutex+ instanceLocks = map[string]*sync.Mutex{}+)++func instanceLock(name string) *sync.Mutex {+ instanceLocksMu.Lock()+ defer instanceLocksMu.Unlock()+ if instanceLocks[name] == nil {+ instanceLocks[name] = &sync.Mutex{}+ }+ return instanceLocks[name]+}++// registerDatabase names a database in the instance's sync list, which an+// instance on the shared target carries so it never imports another test's+// databases, and syncs the instance. An instance with a server to itself keeps+// no list and only syncs.+func (ctl *controller) registerDatabase(ctx context.Context, instance *v1pb.Instance, databaseName string) error {+ mu := instanceLock(instance.Name)+ mu.Lock()+ defer mu.Unlock()++ if instance.GetSyncDatabases() != nil {+ resp, err := ctl.instanceServiceClient.GetInstance(ctx, connect.NewRequest(&v1pb.GetInstanceRequest{Name: instance.Name}))+ if err != nil {+ return err+ }+ if databases := resp.Msg.GetSyncDatabases().GetDatabases(); !slices.Contains(databases, databaseName) {+ if _, err := ctl.instanceServiceClient.UpdateInstance(ctx, connect.NewRequest(&v1pb.UpdateInstanceRequest{+ Instance: &v1pb.Instance{+ Name: instance.Name,+ SyncDatabases: &v1pb.SyncDatabases{Databases: append(databases, databaseName)},+ },+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"sync_databases"}},+ })); err != nil {+ return err+ }+ }+ }+ _, err := ctl.instanceServiceClient.SyncInstance(ctx, connect.NewRequest(&v1pb.SyncInstanceRequest{Name: instance.Name}))+ return err+}diff --git a/backend/tests/environment_test.go b/backend/tests/environment_test.goindex 745faa2a24..e878369d61 100644--- a/backend/tests/environment_test.go+++ b/backend/tests/environment_test.go@@ -40,7 +40,7 @@ func TestDatabaseEnvironment(t *testing.T) {instance := instanceResp.Msgdb0Name := uniqueDB("db0")- err = ctl.createDatabase(ctx, ctl.project, instance, testEnvironment /* environment */, db0Name, "")+ err = ctl.createDatabaseByRollout(ctx, ctl.project, instance, testEnvironment /* environment */, db0Name)a.NoError(err)db0Resp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{Name: fmt.Sprintf("%s/databases/%s", instance.Name, db0Name),@@ -53,7 +53,7 @@ func TestDatabaseEnvironment(t *testing.T) {a.Equal(testEnvironment.Name, *db0.EffectiveEnvironment)db1Name := uniqueDB("db1")- err = ctl.createDatabase(ctx, ctl.project, instance, nil /* environment */, db1Name, "")+ err = ctl.createDatabaseByRollout(ctx, ctl.project, instance, nil /* environment */, db1Name)a.NoError(err)db1Resp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{Name: fmt.Sprintf("%s/databases/%s", instance.Name, db1Name),@@ -65,7 +65,7 @@ func TestDatabaseEnvironment(t *testing.T) {a.Equal(prodEnvironment.Name, *db1.EffectiveEnvironment)db2Name := uniqueDB("db2")- err = ctl.createDatabase(ctx, ctl.project, instance, nil /* environment */, db2Name, "")+ err = ctl.createDatabaseByRollout(ctx, ctl.project, instance, nil /* environment */, db2Name)a.NoError(err)db2Resp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{Name: fmt.Sprintf("%s/databases/%s", instance.Name, db2Name),diff --git a/backend/tests/ghost_test.go b/backend/tests/ghost_test.goindex 9df7c99c6b..17cc974e73 100644--- a/backend/tests/ghost_test.go+++ b/backend/tests/ghost_test.go@@ -79,10 +79,10 @@ func TestGhostSchemaUpdate(t *testing.T) {// Create backup database for MySQL.backupDBName := common.BackupDatabaseNameOfEngine(storepb.Engine_MYSQL)- err = ctl.createDatabase(ctx, ctl.project, instance, nil /* environment */, backupDBName, "")+ err = ctl.createDatabaseByRollout(ctx, ctl.project, instance, nil /* environment */, backupDBName)a.NoError(err)- err = ctl.createDatabase(ctx, ctl.project, instance, nil /* environment */, databaseName, "")+ err = ctl.createDatabaseByRollout(ctx, ctl.project, instance, nil /* environment */, databaseName)a.NoError(err)databaseResponse, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{diff --git a/backend/tests/gitops_test.go b/backend/tests/gitops_test.goindex 1c05b866aa..5718af8a3e 100644--- a/backend/tests/gitops_test.go+++ b/backend/tests/gitops_test.go@@ -816,9 +816,9 @@ func TestGitOpsRolloutGhostDirective(t *testing.T) {instance := instanceResp.MsgbackupDBName := common.BackupDatabaseNameOfEngine(storepb.Engine_MYSQL)- err = ctl.createDatabase(ctx, project, instance, nil, backupDBName, "")+ err = ctl.createDatabaseByRollout(ctx, project, instance, nil, backupDBName)a.NoError(err)- err = ctl.createDatabase(ctx, project, instance, nil, databaseName, "")+ err = ctl.createDatabaseByRollout(ctx, project, instance, nil, databaseName)a.NoError(err)createReleaseResp, err := ctl.releaseServiceClient.CreateRelease(ctx, connect.NewRequest(&v1pb.CreateReleaseRequest{diff --git a/backend/tests/pg_sdl_rollout_test.go b/backend/tests/pg_sdl_rollout_test.goindex 0a9d670718..6bfe2e95c9 100644--- a/backend/tests/pg_sdl_rollout_test.go+++ b/backend/tests/pg_sdl_rollout_test.go@@ -64,10 +64,6 @@ func TestPgSDLRollout(t *testing.T) {// Create unique database namedbName := fmt.Sprintf("sdl_workflow_%s", strings.ReplaceAll(uuid.New().String()[:8], "-", ""))- // Create database directly in PostgreSQL- _, err := pgContainer.GetDB().Exec(fmt.Sprintf("CREATE DATABASE %s", dbName))- a.NoError(err)-// Create database in Bytebaseerr = ctl.createDatabase(ctx, ctl.project, instance, nil, dbName, "postgres")a.NoError(err)@@ -187,10 +183,6 @@ CREATE TABLE "public"."posts" (// Create unique database namedbName := fmt.Sprintf("sdl_empty_%s", strings.ReplaceAll(uuid.New().String()[:8], "-", ""))- // Create database directly in PostgreSQL- _, err := pgContainer.GetDB().Exec(fmt.Sprintf("CREATE DATABASE %s", dbName))- a.NoError(err)-// Create database in Bytebaseerr = ctl.createDatabase(ctx, ctl.project, instance, nil, dbName, "postgres")a.NoError(err)@@ -228,10 +220,6 @@ CREATE TABLE "public"."posts" (// Create unique database namedbName := fmt.Sprintf("sdl_nochange_%s", strings.ReplaceAll(uuid.New().String()[:8], "-", ""))- // Create database directly in PostgreSQL- _, err := pgContainer.GetDB().Exec(fmt.Sprintf("CREATE DATABASE %s", dbName))- a.NoError(err)-// Create database in Bytebaseerr = ctl.createDatabase(ctx, ctl.project, instance, nil, dbName, "postgres")a.NoError(err)diff --git a/backend/tests/rollout.go b/backend/tests/rollout.goindex dd1e9964d9..3930ef9cf1 100644--- a/backend/tests/rollout.go+++ b/backend/tests/rollout.go@@ -71,9 +71,13 @@ func (ctl *controller) changeDatabaseWithConfigAndReturnRollout(ctx context.Contreturn rollout, nil}+const pollInterval = 100 * time.Millisecond+// waitRollout waits for pipeline to finish and approves tasks when necessary.+// Both loops read before they wait: a ticker in front of the first read charges+// its interval to every caller, and this waits twice.func (ctl *controller) waitRollout(ctx context.Context, issueName, rolloutName string) error {- ticker := time.NewTicker(300 * time.Millisecond)+ ticker := time.NewTicker(pollInterval)defer ticker.Stop()// Add timeout to prevent infinite loops@@ -83,35 +87,28 @@ func (ctl *controller) waitRollout(ctx context.Context, issueName, rolloutName s// Wait for approvalwaitApproval:for {+ issueResp, err := ctl.issueServiceClient.GetIssue(ctx, connect.NewRequest(&v1pb.GetIssueRequest{Name: issueName}))+ if err != nil {+ return err+ }+ issue := issueResp.Msg+ if issue.ApprovalStatus == v1pb.ApprovalStatus_APPROVED || issue.ApprovalStatus == v1pb.ApprovalStatus_SKIPPED {+ break waitApproval+ }+ // If the issue is pending approval, approve it (test user should be project owner)+ if issue.ApprovalStatus == v1pb.ApprovalStatus_PENDING {+ if _, err := ctl.issueServiceClient.ApproveIssue(ctx, connect.NewRequest(&v1pb.ApproveIssueRequest{+ Name: issueName,+ })); err != nil {+ return errors.Wrapf(err, "failed to approve issue")+ }+ }+select {case <-timeout:- // Timeout - fetch current state for debugging- issueResp, err := ctl.issueServiceClient.GetIssue(ctx, connect.NewRequest(&v1pb.GetIssueRequest{Name: issueName}))- if err != nil {- return errors.Wrapf(err, "timeout after %v waiting for approval (failed to fetch issue state)", time.Since(startTime))- }- issue := issueResp.Msgreturn errors.Errorf("timeout after %v waiting for approval to complete, current approval status: %s",time.Since(startTime), issue.ApprovalStatus.String())-case <-ticker.C:- issueResp, err := ctl.issueServiceClient.GetIssue(ctx, connect.NewRequest(&v1pb.GetIssueRequest{Name: issueName}))- if err != nil {- return err- }- issue := issueResp.Msg- if issue.ApprovalStatus == v1pb.ApprovalStatus_APPROVED || issue.ApprovalStatus == v1pb.ApprovalStatus_SKIPPED {- break waitApproval- }- // If the issue is pending approval, approve it (test user should be project owner)- if issue.ApprovalStatus == v1pb.ApprovalStatus_PENDING {- _, err := ctl.issueServiceClient.ApproveIssue(ctx, connect.NewRequest(&v1pb.ApproveIssueRequest{- Name: issueName,- }))- if err != nil {- return errors.Wrapf(err, "failed to approve issue")- }- }}}@@ -140,7 +137,12 @@ waitApproval:}}- for range ticker.C {+ for {+ select {+ case <-timeout:+ return errors.Errorf("timeout after %v waiting for rollout to complete", time.Since(startTime))+ default:+ }rolloutResp, err := ctl.rolloutServiceClient.GetRollout(ctx, connect.NewRequest(&v1pb.GetRolloutRequest{Name: rolloutName,}))@@ -186,16 +188,16 @@ waitApproval:}if completed {- break+ return nil}+ <-ticker.C}- return nil}// waitRolloutWithoutApproval waits for a rollout to complete without going through the issue approval flow.// This is used for test scenarios where issues are not created (e.g., direct rollout creation).func (ctl *controller) waitRolloutWithoutApproval(ctx context.Context, rolloutName string) error {- ticker := time.NewTicker(300 * time.Millisecond)+ ticker := time.NewTicker(pollInterval)defer ticker.Stop()// Add timeout to prevent infinite loops@@ -228,16 +230,55 @@ func (ctl *controller) waitRolloutWithoutApproval(ctx context.Context, rolloutNa}for {+ rolloutResp, err := ctl.rolloutServiceClient.GetRollout(ctx, connect.NewRequest(&v1pb.GetRolloutRequest{+ Name: rolloutName,+ }))+ if err != nil {+ return err+ }+ rollout := rolloutResp.Msg+ completed := true+ var runTasks []string+ for _, stage := range rollout.Stages {+ for _, task := range stage.Tasks {+ switch task.Status {+ case v1pb.Task_NOT_STARTED:+ runTasks = append(runTasks, task.Name)+ completed = false+ case v1pb.Task_DONE:+ continue+ case v1pb.Task_SKIPPED:+ continue+ case v1pb.Task_FAILED:+ resp, err := ctl.rolloutServiceClient.ListTaskRuns(ctx, connect.NewRequest(&v1pb.ListTaskRunsRequest{Parent: task.Name}))+ if err != nil {+ return err+ }+ if len(resp.Msg.TaskRuns) > 0 {+ return errors.New(resp.Msg.TaskRuns[0].Detail)+ }+ default:+ completed = false+ }+ }+ }++ // Rollout tasks.+ if len(runTasks) > 0 {+ if _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{+ Parent: fmt.Sprintf("%s/stages/-", rolloutName),+ Tasks: runTasks,+ })); err != nil {+ return err+ }+ }++ if completed {+ return nil+ }+select {case <-timeout:- // Timeout - fetch current state for debugging- rolloutResp, err := ctl.rolloutServiceClient.GetRollout(ctx, connect.NewRequest(&v1pb.GetRolloutRequest{- Name: rolloutName,- }))- if err != nil {- return errors.Wrapf(err, "timeout after %v waiting for rollout (failed to fetch rollout state)", time.Since(startTime))- }- rollout := rolloutResp.Msgvar taskStatuses []stringfor _, stage := range rollout.Stages {for _, task := range stage.Tasks {@@ -246,55 +287,7 @@ func (ctl *controller) waitRolloutWithoutApproval(ctx context.Context, rolloutNa}return errors.Errorf("timeout after %v waiting for rollout to complete, task statuses: %s",time.Since(startTime), strings.Join(taskStatuses, ", "))-case <-ticker.C:- rolloutResp, err := ctl.rolloutServiceClient.GetRollout(ctx, connect.NewRequest(&v1pb.GetRolloutRequest{- Name: rolloutName,- }))- if err != nil {- return err- }- rollout := rolloutResp.Msg- completed := true- var runTasks []string- for _, stage := range rollout.Stages {- for _, task := range stage.Tasks {- switch task.Status {- case v1pb.Task_NOT_STARTED:- runTasks = append(runTasks, task.Name)- completed = false- case v1pb.Task_DONE:- continue- case v1pb.Task_SKIPPED:- continue- case v1pb.Task_FAILED:- resp, err := ctl.rolloutServiceClient.ListTaskRuns(ctx, connect.NewRequest(&v1pb.ListTaskRunsRequest{Parent: task.Name}))- if err != nil {- return err- }- if len(resp.Msg.TaskRuns) > 0 {- return errors.New(resp.Msg.TaskRuns[0].Detail)- }- default:- completed = false- }- }- }-- // Rollout tasks.- if len(runTasks) > 0 {- _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{- Parent: fmt.Sprintf("%s/stages/-", rolloutName),- Tasks: runTasks,- }))- if err != nil {- return err- }- }-- if completed {- return nil- }}}}diff --git a/backend/tests/testcontainer.go b/backend/tests/testcontainer.goindex 75296ac0d7..268df519a5 100644--- a/backend/tests/testcontainer.go+++ b/backend/tests/testcontainer.go@@ -2,8 +2,11 @@ package testsimport ("context"+ "net"+ "sync""testing"+ "github.com/pkg/errors""github.com/stretchr/testify/require""github.com/bytebase/bytebase/backend/common/testcontainer"@@ -16,13 +19,31 @@ type Container struct {*testcontainer.Container}+// pgContainers maps each Postgres the tests hold, by host:port, to its+// container, so createDatabase can reach the server behind an instance.+var (+ pgContainersMu sync.Mutex+ pgContainers = map[string]*Container{}+)++func pgContainerOf(instance *v1pb.Instance) (*Container, error) {+ pgContainersMu.Lock()+ defer pgContainersMu.Unlock()+ for _, ds := range instance.GetDataSources() {+ if c, ok := pgContainers[net.JoinHostPort(ds.Host, ds.Port)]; ok && ds.Type == v1pb.DataSourceType_ADMIN {+ return c, nil+ }+ }+ return nil, errors.Errorf("instance %s is not on a test Postgres; use createDatabaseByRollout", instance.Name)+}+// provisionPgInstance starts a Postgres of the test's own, closed when the test// finishes. Use it for what sharedPgTarget cannot serve: a cluster-wide write// such as a role, two instances that must be two servers, or a connection the// test breaks on purpose.func provisionPgInstance(t *testing.T) *Container {t.Helper()- return provision(t, testcontainer.GetPgContainer)+ return registerPg(provision(t, testcontainer.GetPgContainer))}// dataSource returns a data source of the given type and ID pointing at the container.@@ -48,8 +69,14 @@ func (c *Container) adminDataSource() *v1pb.DataSource {// test's databases too.func sharedPgTarget(t *testing.T) *Container {t.Helper()- tc := testcontainer.SharedTargetPgContainer(t)- return &Container{tc}+ return registerPg(&Container{testcontainer.SharedTargetPgContainer(t)})+}++func registerPg(c *Container) *Container {+ pgContainersMu.Lock()+ defer pgContainersMu.Unlock()+ pgContainers[net.JoinHostPort(c.GetHost(), c.GetPort())] = c+ return c}// provisionMySQLInstance is provisionPgInstance for MySQL.diff --git a/backend/tests/webhook_helpers_test.go b/backend/tests/webhook_helpers_test.goindex f15ca2f58e..e0bc48ca83 100644--- a/backend/tests/webhook_helpers_test.go+++ b/backend/tests/webhook_helpers_test.go@@ -81,46 +81,27 @@ const webhookWaitTimeout = 30 * time.Second// waitForWebhookCount blocks until at least n webhooks for (project, eventTitle)// have arrived, or fails the test after webhookWaitTimeout.-func waitForWebhookCount(t *testing.T, c *webhookCollector, projectName, eventTitle string, n int, resources ...string) {+func waitForWebhookCount(t *testing.T, c *webhookCollector, projectName, eventTitle string, n int) {t.Helper()deadline := time.Now().Add(webhookWaitTimeout)for {- count := countWebhooksFor(c, projectName, eventTitle, resources...)+ count := countWebhooksFor(c, projectName, eventTitle)if count >= n {return}if time.Now().After(deadline) {- t.Fatalf("timed out waiting for %d %q webhooks on %s for resources %v; got %d after %s",- n, eventTitle, projectName, resources, count, webhookWaitTimeout)+ t.Fatalf("timed out waiting for %d %q webhooks on %s; got %d after %s",+ n, eventTitle, projectName, count, webhookWaitTimeout)}time.Sleep(100 * time.Millisecond)}}-// createDatabasesFlushingCompletion registers a PIPELINE_FAILED and-// PIPELINE_COMPLETED webhook, provisions the databases, then waits for the-// completion each creation rollout fires and discards it. Registering the-// webhook after creation is not enough: the scheduler marks the task DONE,-// which is what createDatabase waits on, before it emits the completion event,-// so a webhook added in that gap still receives it and a later assertion of-// zero completions fails.-func createDatabasesFlushingCompletion(ctx context.Context, t *testing.T, ctl *controller, c *webhookCollector, project *v1pb.Project, instance *v1pb.Instance, url string, databases ...string) {- t.Helper()- addWebhookForEvents(ctx, t, ctl, project, url, []v1pb.Activity_Type{- v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,- })- for _, database := range databases {- require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, database, ""))- }- waitForWebhookCount(t, c, project.Name, "Rollout completed", len(databases))- c.reset()-}-// requireWebhookCount asserts the exact count of webhooks for (project, eventTitle).-func requireWebhookCount(t *testing.T, c *webhookCollector, projectName, eventTitle string, n int, resources ...string) {+func requireWebhookCount(t *testing.T, c *webhookCollector, projectName, eventTitle string, n int) {t.Helper()- got := countWebhooksFor(c, projectName, eventTitle, resources...)- require.Equalf(t, n, got, "expected %d %q webhooks on %s for resources %v, got %d", n, eventTitle, projectName, resources, got)+ got := countWebhooksFor(c, projectName, eventTitle)+ require.Equalf(t, n, got, "expected %d %q webhooks on %s, got %d", n, eventTitle, projectName, got)}func countWebhooksFor(c *webhookCollector, projectName, eventTitle string, resources ...string) int {@@ -307,49 +288,6 @@ func skipTaskByDB(ctx context.Context, t *testing.T, ctl *controller, rollout *vrequire.NoError(t, err)}-// skipFailedTasks finds every FAILED task in the rollout and skips them.-func skipFailedTasks(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout) {- t.Helper()- fresh := refreshRollout(ctx, t, ctl, rollout)- perStage := map[string][]string{}- for _, stage := range fresh.Stages {- for _, task := range stage.Tasks {- if task.Status == v1pb.Task_FAILED {- perStage[stage.Name] = append(perStage[stage.Name], task.Name)- }- }- }- require.NotEmpty(t, perStage, "expected at least one failed task to skip")- for stageName, names := range perStage {- _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{- Parent: stageName,- Tasks: names,- Reason: "test: skip failed tasks",- }))- require.NoError(t, err)- }-}--func skipAllTasks(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout) {- t.Helper()- fresh := refreshRollout(ctx, t, ctl, rollout)- for _, stage := range fresh.Stages {- var names []string- for _, task := range stage.Tasks {- names = append(names, task.Name)- }- if len(names) == 0 {- continue- }- _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{- Parent: stage.Name,- Tasks: names,- Reason: "test: skip all",- }))- require.NoError(t, err)- }-}-// retryFailedTasks reruns BatchRunTasks on every FAILED task. Caller decides// whether to call unblockFailingTask first (to make the retry pass) or not// (to test that BatchRunTasks resets the dedup row so a seconddiff --git a/backend/tests/webhook_test.go b/backend/tests/webhook_test.goindex 25177eefa3..9e8de9c0a8 100644--- a/backend/tests/webhook_test.go+++ b/backend/tests/webhook_test.go@@ -161,7 +161,8 @@ func TestWebhookIntegration(t *testing.T) {w.WriteHeader(http.StatusOK)_, _ = w.Write([]byte("ok"))}))- defer webhookServer.Close()+ // Cleanup, not defer: the parallel subtests below run after this body returns.+ t.Cleanup(webhookServer.Close)// Create a single instance for all testspgContainer := sharedPgTarget(t)@@ -420,34 +421,12 @@ func TestWebhookIntegration(t *testing.T) {require.True(t, foundCorrectWebhook, "expected a webhook whose Slack description matches the plan description")})- t.Run("PipelineCompletedAfterSkippingFailedTask", func(t *testing.T) {- collector.reset()-- project := ctl.createTestProject(ctx, t, "byt9398-c4")-- createDatabasesFlushingCompletion(ctx, t, ctl, collector, project, instance, webhookServer.URL, "byt9398_c4_pass", "byt9398_c4_fail")-- plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{- {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c4_pass")},- {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c4_fail")},- })- rollout := runAllTasks(ctx, t, ctl, plan)-- // Phase 1: failing task → exactly one PIPELINE_FAILED, no PIPELINE_COMPLETED.- waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1, rollout.Name)- waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)- requireWebhookCount(t, collector, project.Name, "Rollout completed", 0, rollout.Name)-- // Phase 2: skip the failed task → PIPELINE_COMPLETED fires (the fix).- skipFailedTasks(ctx, t, ctl, rollout)- waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1, rollout.Name)- requireWebhookCount(t, collector, project.Name, "Rollout failed", 1, rollout.Name)- })-t.Run("PipelineCompleted_AllTasksDone", func(t *testing.T) {- collector.reset()+ t.Parallel()project := ctl.createTestProject(ctx, t, "byt9398-c1")- createDatabasesFlushingCompletion(ctx, t, ctl, collector, project, instance, webhookServer.URL, "byt9398_c1_a", "byt9398_c1_b")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c1_a", ""))+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c1_b", ""))+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED})plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c1_a")},@@ -459,84 +438,13 @@ func TestWebhookIntegration(t *testing.T) {requireWebhookCount(t, collector, project.Name, "Rollout failed", 0)})- t.Run("PipelineCompleted_DoneAndSkipped", func(t *testing.T) {- collector.reset()- project := ctl.createTestProject(ctx, t, "byt9398-c2")- createDatabasesFlushingCompletion(ctx, t, ctl, collector, project, instance, webhookServer.URL, "byt9398_c2_a", "byt9398_c2_b")-- plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{- {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c2_a")},- {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c2_b")},- })- rollout := createRolloutOnly(ctx, t, ctl, plan)- skipTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c2_b"))- runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c2_a"))-- waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)- requireWebhookCount(t, collector, project.Name, "Rollout failed", 0)- })-- t.Run("PipelineCompleted_AllSkipped", func(t *testing.T) {- collector.reset()- project := ctl.createTestProject(ctx, t, "byt9398-c5")- createDatabasesFlushingCompletion(ctx, t, ctl, collector, project, instance, webhookServer.URL, "byt9398_c5_a", "byt9398_c5_b")-- plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{- {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c5_a")},- {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c5_b")},- })- rollout := createRolloutOnly(ctx, t, ctl, plan)- skipAllTasks(ctx, t, ctl, rollout)-- waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)- requireWebhookCount(t, collector, project.Name, "Rollout failed", 0)- })-- t.Run("PipelineCompleted_DoneAndFailedThenRetriedDone", func(t *testing.T) {- collector.reset()- project := ctl.createTestProject(ctx, t, "byt9398-c3")- createDatabasesFlushingCompletion(ctx, t, ctl, collector, project, instance, webhookServer.URL, "byt9398_c3_pass", "byt9398_c3_fail")-- plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{- {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c3_pass")},- {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c3_fail")},- })- rollout := runAllTasks(ctx, t, ctl, plan)-- waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)- waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)- requireWebhookCount(t, collector, project.Name, "Rollout completed", 0)-- unblockFailingTask(t, pgContainer, "byt9398_c3_fail")- retryFailedTasks(ctx, t, ctl, rollout)-- waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)- requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)- })-- t.Run("PipelineCompleted_AllFailedThenAllSkipped", func(t *testing.T) {- collector.reset()- project := ctl.createTestProject(ctx, t, "byt9398-c6")- createDatabasesFlushingCompletion(ctx, t, ctl, collector, project, instance, webhookServer.URL, "byt9398_c6_a", "byt9398_c6_b")-- plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{- {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c6_a")},- {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c6_b")},- })- rollout := runAllTasks(ctx, t, ctl, plan)-- waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)- waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)-- skipAllTasks(ctx, t, ctl, rollout)- waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)- requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)- })-t.Run("PipelineCompleted_MixedRecovery", func(t *testing.T) {- collector.reset()+ t.Parallel()project := ctl.createTestProject(ctx, t, "byt9398-c7")- createDatabasesFlushingCompletion(ctx, t, ctl, collector, project, instance, webhookServer.URL, "byt9398_c7_done", "byt9398_c7_skip", "byt9398_c7_retry", "byt9398_c7_skipfailed")+ for _, database := range []string{"byt9398_c7_done", "byt9398_c7_skip", "byt9398_c7_retry", "byt9398_c7_skipfailed"} {+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, database, ""))+ }+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED})plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_done")},@@ -567,10 +475,9 @@ func TestWebhookIntegration(t *testing.T) {})t.Run("PipelineFailed_SingleTaskFails", func(t *testing.T) {- collector.reset()+ t.Parallel()project := ctl.createTestProject(ctx, t, "byt9398-f1")require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f1_fail", ""))- collector.reset()addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED})plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{{seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f1_fail")},@@ -579,32 +486,10 @@ func TestWebhookIntegration(t *testing.T) {waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)})- t.Run("PipelineFailed_DedupOnSecondTaskFailure", func(t *testing.T) {- collector.reset()- project := ctl.createTestProject(ctx, t, "byt9398-f2")- require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f2_a", ""))- require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f2_b", ""))- collector.reset()- addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED})- plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{- {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f2_a")},- {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f2_b")},- })- rollout := runAllTasks(ctx, t, ctl, plan)-- waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)- waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)-- // Both tasks have failed. ClaimPipelineFailureNotification's PK collision- // must dedupe — assert exactly 1 even after both terminal.- requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)- })-t.Run("PipelineFailed_RetryFailsAgain", func(t *testing.T) {- collector.reset()+ t.Parallel()project := ctl.createTestProject(ctx, t, "byt9398-f3")require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f3_fail", ""))- collector.reset()addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED})plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{{seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f3_fail")},@@ -630,7 +515,7 @@ func TestWebhookIntegration(t *testing.T) {installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})appr := provisionApprover(ctx, t, ctl, project, "i2", "roles/projectOwner")- collector.reset() // flush any creation-time webhook events+ collector.reset()addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_CREATED})plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{